719 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright(C) 2016 Linaro Limited. All rights reserved.4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>5 */6 7#include <linux/atomic.h>8#include <linux/circ_buf.h>9#include <linux/coresight.h>10#include <linux/perf_event.h>11#include <linux/slab.h>12#include "coresight-priv.h"13#include "coresight-tmc.h"14#include "coresight-etm-perf.h"15 16static int tmc_set_etf_buffer(struct coresight_device *csdev,17 struct perf_output_handle *handle);18 19static int __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)20{21 int rc = 0;22 23 CS_UNLOCK(drvdata->base);24 25 /* Wait for TMCSReady bit to be set */26 rc = tmc_wait_for_tmcready(drvdata);27 if (rc) {28 dev_err(&drvdata->csdev->dev,29 "Failed to enable: TMC not ready\n");30 CS_LOCK(drvdata->base);31 return rc;32 }33 34 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);35 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |36 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |37 TMC_FFCR_TRIGON_TRIGIN,38 drvdata->base + TMC_FFCR);39 40 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);41 tmc_enable_hw(drvdata);42 43 CS_LOCK(drvdata->base);44 return rc;45}46 47static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)48{49 int rc = coresight_claim_device(drvdata->csdev);50 51 if (rc)52 return rc;53 54 rc = __tmc_etb_enable_hw(drvdata);55 if (rc)56 coresight_disclaim_device(drvdata->csdev);57 return rc;58}59 60static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)61{62 char *bufp;63 u32 read_data, lost;64 65 /* Check if the buffer wrapped around. */66 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;67 bufp = drvdata->buf;68 drvdata->len = 0;69 while (1) {70 read_data = readl_relaxed(drvdata->base + TMC_RRD);71 if (read_data == 0xFFFFFFFF)72 break;73 memcpy(bufp, &read_data, 4);74 bufp += 4;75 drvdata->len += 4;76 }77 78 if (lost)79 coresight_insert_barrier_packet(drvdata->buf);80 return;81}82 83static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)84{85 CS_UNLOCK(drvdata->base);86 87 tmc_flush_and_stop(drvdata);88 /*89 * When operating in sysFS mode the content of the buffer needs to be90 * read before the TMC is disabled.91 */92 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)93 tmc_etb_dump_hw(drvdata);94 tmc_disable_hw(drvdata);95 96 CS_LOCK(drvdata->base);97}98 99static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)100{101 __tmc_etb_disable_hw(drvdata);102 coresight_disclaim_device(drvdata->csdev);103}104 105static int __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)106{107 int rc = 0;108 109 CS_UNLOCK(drvdata->base);110 111 /* Wait for TMCSReady bit to be set */112 rc = tmc_wait_for_tmcready(drvdata);113 if (rc) {114 dev_err(&drvdata->csdev->dev,115 "Failed to enable : TMC is not ready\n");116 CS_LOCK(drvdata->base);117 return rc;118 }119 120 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);121 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,122 drvdata->base + TMC_FFCR);123 writel_relaxed(0x0, drvdata->base + TMC_BUFWM);124 tmc_enable_hw(drvdata);125 126 CS_LOCK(drvdata->base);127 return rc;128}129 130static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)131{132 int rc = coresight_claim_device(drvdata->csdev);133 134 if (rc)135 return rc;136 137 rc = __tmc_etf_enable_hw(drvdata);138 if (rc)139 coresight_disclaim_device(drvdata->csdev);140 return rc;141}142 143static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)144{145 struct coresight_device *csdev = drvdata->csdev;146 147 CS_UNLOCK(drvdata->base);148 149 tmc_flush_and_stop(drvdata);150 tmc_disable_hw(drvdata);151 coresight_disclaim_device_unlocked(csdev);152 CS_LOCK(drvdata->base);153}154 155/*156 * Return the available trace data in the buffer from @pos, with157 * a maximum limit of @len, updating the @bufpp on where to158 * find it.159 */160ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,161 loff_t pos, size_t len, char **bufpp)162{163 ssize_t actual = len;164 165 /* Adjust the len to available size @pos */166 if (pos + actual > drvdata->len)167 actual = drvdata->len - pos;168 if (actual > 0)169 *bufpp = drvdata->buf + pos;170 return actual;171}172 173static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)174{175 int ret = 0;176 bool used = false;177 char *buf = NULL;178 unsigned long flags;179 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);180 181 /*182 * If we don't have a buffer release the lock and allocate memory.183 * Otherwise keep the lock and move along.184 */185 spin_lock_irqsave(&drvdata->spinlock, flags);186 if (!drvdata->buf) {187 spin_unlock_irqrestore(&drvdata->spinlock, flags);188 189 /* Allocating the memory here while outside of the spinlock */190 buf = kzalloc(drvdata->size, GFP_KERNEL);191 if (!buf)192 return -ENOMEM;193 194 /* Let's try again */195 spin_lock_irqsave(&drvdata->spinlock, flags);196 }197 198 if (drvdata->reading) {199 ret = -EBUSY;200 goto out;201 }202 203 /*204 * In sysFS mode we can have multiple writers per sink. Since this205 * sink is already enabled no memory is needed and the HW need not be206 * touched.207 */208 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {209 csdev->refcnt++;210 goto out;211 }212 213 /*214 * If drvdata::buf isn't NULL, memory was allocated for a previous215 * trace run but wasn't read. If so simply zero-out the memory.216 * Otherwise use the memory allocated above.217 *218 * The memory is freed when users read the buffer using the219 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for220 * details.221 */222 if (drvdata->buf) {223 memset(drvdata->buf, 0, drvdata->size);224 } else {225 used = true;226 drvdata->buf = buf;227 }228 229 ret = tmc_etb_enable_hw(drvdata);230 if (!ret) {231 coresight_set_mode(csdev, CS_MODE_SYSFS);232 csdev->refcnt++;233 } else {234 /* Free up the buffer if we failed to enable */235 used = false;236 }237out:238 spin_unlock_irqrestore(&drvdata->spinlock, flags);239 240 /* Free memory outside the spinlock if need be */241 if (!used)242 kfree(buf);243 244 return ret;245}246 247static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)248{249 int ret = 0;250 pid_t pid;251 unsigned long flags;252 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);253 struct perf_output_handle *handle = data;254 struct cs_buffers *buf = etm_perf_sink_config(handle);255 256 spin_lock_irqsave(&drvdata->spinlock, flags);257 do {258 ret = -EINVAL;259 if (drvdata->reading)260 break;261 /*262 * No need to continue if the ETB/ETF is already operated263 * from sysFS.264 */265 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {266 ret = -EBUSY;267 break;268 }269 270 /* Get a handle on the pid of the process to monitor */271 pid = buf->pid;272 273 if (drvdata->pid != -1 && drvdata->pid != pid) {274 ret = -EBUSY;275 break;276 }277 278 ret = tmc_set_etf_buffer(csdev, handle);279 if (ret)280 break;281 282 /*283 * No HW configuration is needed if the sink is already in284 * use for this session.285 */286 if (drvdata->pid == pid) {287 csdev->refcnt++;288 break;289 }290 291 ret = tmc_etb_enable_hw(drvdata);292 if (!ret) {293 /* Associate with monitored process. */294 drvdata->pid = pid;295 coresight_set_mode(csdev, CS_MODE_PERF);296 csdev->refcnt++;297 }298 } while (0);299 spin_unlock_irqrestore(&drvdata->spinlock, flags);300 301 return ret;302}303 304static int tmc_enable_etf_sink(struct coresight_device *csdev,305 enum cs_mode mode, void *data)306{307 int ret;308 309 switch (mode) {310 case CS_MODE_SYSFS:311 ret = tmc_enable_etf_sink_sysfs(csdev);312 break;313 case CS_MODE_PERF:314 ret = tmc_enable_etf_sink_perf(csdev, data);315 break;316 /* We shouldn't be here */317 default:318 ret = -EINVAL;319 break;320 }321 322 if (ret)323 return ret;324 325 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");326 return 0;327}328 329static int tmc_disable_etf_sink(struct coresight_device *csdev)330{331 unsigned long flags;332 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);333 334 spin_lock_irqsave(&drvdata->spinlock, flags);335 336 if (drvdata->reading) {337 spin_unlock_irqrestore(&drvdata->spinlock, flags);338 return -EBUSY;339 }340 341 csdev->refcnt--;342 if (csdev->refcnt) {343 spin_unlock_irqrestore(&drvdata->spinlock, flags);344 return -EBUSY;345 }346 347 /* Complain if we (somehow) got out of sync */348 WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);349 tmc_etb_disable_hw(drvdata);350 /* Dissociate from monitored process. */351 drvdata->pid = -1;352 coresight_set_mode(csdev, CS_MODE_DISABLED);353 354 spin_unlock_irqrestore(&drvdata->spinlock, flags);355 356 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");357 return 0;358}359 360static int tmc_enable_etf_link(struct coresight_device *csdev,361 struct coresight_connection *in,362 struct coresight_connection *out)363{364 int ret = 0;365 unsigned long flags;366 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);367 bool first_enable = false;368 369 spin_lock_irqsave(&drvdata->spinlock, flags);370 if (drvdata->reading) {371 spin_unlock_irqrestore(&drvdata->spinlock, flags);372 return -EBUSY;373 }374 375 if (csdev->refcnt == 0) {376 ret = tmc_etf_enable_hw(drvdata);377 if (!ret) {378 coresight_set_mode(csdev, CS_MODE_SYSFS);379 first_enable = true;380 }381 }382 if (!ret)383 csdev->refcnt++;384 spin_unlock_irqrestore(&drvdata->spinlock, flags);385 386 if (first_enable)387 dev_dbg(&csdev->dev, "TMC-ETF enabled\n");388 return ret;389}390 391static void tmc_disable_etf_link(struct coresight_device *csdev,392 struct coresight_connection *in,393 struct coresight_connection *out)394{395 unsigned long flags;396 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);397 bool last_disable = false;398 399 spin_lock_irqsave(&drvdata->spinlock, flags);400 if (drvdata->reading) {401 spin_unlock_irqrestore(&drvdata->spinlock, flags);402 return;403 }404 405 csdev->refcnt--;406 if (csdev->refcnt == 0) {407 tmc_etf_disable_hw(drvdata);408 coresight_set_mode(csdev, CS_MODE_DISABLED);409 last_disable = true;410 }411 spin_unlock_irqrestore(&drvdata->spinlock, flags);412 413 if (last_disable)414 dev_dbg(&csdev->dev, "TMC-ETF disabled\n");415}416 417static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,418 struct perf_event *event, void **pages,419 int nr_pages, bool overwrite)420{421 int node;422 struct cs_buffers *buf;423 424 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);425 426 /* Allocate memory structure for interaction with Perf */427 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);428 if (!buf)429 return NULL;430 431 buf->pid = task_pid_nr(event->owner);432 buf->snapshot = overwrite;433 buf->nr_pages = nr_pages;434 buf->data_pages = pages;435 436 return buf;437}438 439static void tmc_free_etf_buffer(void *config)440{441 struct cs_buffers *buf = config;442 443 kfree(buf);444}445 446static int tmc_set_etf_buffer(struct coresight_device *csdev,447 struct perf_output_handle *handle)448{449 int ret = 0;450 unsigned long head;451 struct cs_buffers *buf = etm_perf_sink_config(handle);452 453 if (!buf)454 return -EINVAL;455 456 /* wrap head around to the amount of space we have */457 head = handle->head & (((unsigned long)buf->nr_pages << PAGE_SHIFT) - 1);458 459 /* find the page to write to */460 buf->cur = head / PAGE_SIZE;461 462 /* and offset within that page */463 buf->offset = head % PAGE_SIZE;464 465 local_set(&buf->data_size, 0);466 467 return ret;468}469 470static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,471 struct perf_output_handle *handle,472 void *sink_config)473{474 bool lost = false;475 int i, cur;476 const u32 *barrier;477 u32 *buf_ptr;478 u64 read_ptr, write_ptr;479 u32 status;480 unsigned long offset, to_read = 0, flags;481 struct cs_buffers *buf = sink_config;482 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);483 484 if (!buf)485 return 0;486 487 /* This shouldn't happen */488 if (WARN_ON_ONCE(coresight_get_mode(csdev) != CS_MODE_PERF))489 return 0;490 491 spin_lock_irqsave(&drvdata->spinlock, flags);492 493 /* Don't do anything if another tracer is using this sink */494 if (csdev->refcnt != 1)495 goto out;496 497 CS_UNLOCK(drvdata->base);498 499 tmc_flush_and_stop(drvdata);500 501 read_ptr = tmc_read_rrp(drvdata);502 write_ptr = tmc_read_rwp(drvdata);503 504 /*505 * Get a hold of the status register and see if a wrap around506 * has occurred. If so adjust things accordingly.507 */508 status = readl_relaxed(drvdata->base + TMC_STS);509 if (status & TMC_STS_FULL) {510 lost = true;511 to_read = drvdata->size;512 } else {513 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);514 }515 516 /*517 * The TMC RAM buffer may be bigger than the space available in the518 * perf ring buffer (handle->size). If so advance the RRP so that we519 * get the latest trace data. In snapshot mode none of that matters520 * since we are expected to clobber stale data in favour of the latest521 * traces.522 */523 if (!buf->snapshot && to_read > handle->size) {524 u32 mask = tmc_get_memwidth_mask(drvdata);525 526 /*527 * Make sure the new size is aligned in accordance with the528 * requirement explained in function tmc_get_memwidth_mask().529 */530 to_read = handle->size & mask;531 /* Move the RAM read pointer up */532 read_ptr = (write_ptr + drvdata->size) - to_read;533 /* Make sure we are still within our limits */534 if (read_ptr > (drvdata->size - 1))535 read_ptr -= drvdata->size;536 /* Tell the HW */537 tmc_write_rrp(drvdata, read_ptr);538 lost = true;539 }540 541 /*542 * Don't set the TRUNCATED flag in snapshot mode because 1) the543 * captured buffer is expected to be truncated and 2) a full buffer544 * prevents the event from being re-enabled by the perf core,545 * resulting in stale data being send to user space.546 */547 if (!buf->snapshot && lost)548 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);549 550 cur = buf->cur;551 offset = buf->offset;552 barrier = coresight_barrier_pkt;553 554 /* for every byte to read */555 for (i = 0; i < to_read; i += 4) {556 buf_ptr = buf->data_pages[cur] + offset;557 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);558 559 if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) {560 *buf_ptr = *barrier;561 barrier++;562 }563 564 offset += 4;565 if (offset >= PAGE_SIZE) {566 offset = 0;567 cur++;568 /* wrap around at the end of the buffer */569 cur &= buf->nr_pages - 1;570 }571 }572 573 /*574 * In snapshot mode we simply increment the head by the number of byte575 * that were written. User space will figure out how many bytes to get576 * from the AUX buffer based on the position of the head.577 */578 if (buf->snapshot)579 handle->head += to_read;580 581 /*582 * CS_LOCK() contains mb() so it can ensure visibility of the AUX trace583 * data before the aux_head is updated via perf_aux_output_end(), which584 * is expected by the perf ring buffer.585 */586 CS_LOCK(drvdata->base);587out:588 spin_unlock_irqrestore(&drvdata->spinlock, flags);589 590 return to_read;591}592 593static const struct coresight_ops_sink tmc_etf_sink_ops = {594 .enable = tmc_enable_etf_sink,595 .disable = tmc_disable_etf_sink,596 .alloc_buffer = tmc_alloc_etf_buffer,597 .free_buffer = tmc_free_etf_buffer,598 .update_buffer = tmc_update_etf_buffer,599};600 601static const struct coresight_ops_link tmc_etf_link_ops = {602 .enable = tmc_enable_etf_link,603 .disable = tmc_disable_etf_link,604};605 606const struct coresight_ops tmc_etb_cs_ops = {607 .sink_ops = &tmc_etf_sink_ops,608};609 610const struct coresight_ops tmc_etf_cs_ops = {611 .sink_ops = &tmc_etf_sink_ops,612 .link_ops = &tmc_etf_link_ops,613};614 615int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)616{617 enum tmc_mode mode;618 int ret = 0;619 unsigned long flags;620 621 /* config types are set a boot time and never change */622 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&623 drvdata->config_type != TMC_CONFIG_TYPE_ETF))624 return -EINVAL;625 626 spin_lock_irqsave(&drvdata->spinlock, flags);627 628 if (drvdata->reading) {629 ret = -EBUSY;630 goto out;631 }632 633 /* Don't interfere if operated from Perf */634 if (coresight_get_mode(drvdata->csdev) == CS_MODE_PERF) {635 ret = -EINVAL;636 goto out;637 }638 639 /* If drvdata::buf is NULL the trace data has been read already */640 if (drvdata->buf == NULL) {641 ret = -EINVAL;642 goto out;643 }644 645 /* Disable the TMC if need be */646 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {647 /* There is no point in reading a TMC in HW FIFO mode */648 mode = readl_relaxed(drvdata->base + TMC_MODE);649 if (mode != TMC_MODE_CIRCULAR_BUFFER) {650 ret = -EINVAL;651 goto out;652 }653 __tmc_etb_disable_hw(drvdata);654 }655 656 drvdata->reading = true;657out:658 spin_unlock_irqrestore(&drvdata->spinlock, flags);659 660 return ret;661}662 663int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)664{665 char *buf = NULL;666 enum tmc_mode mode;667 unsigned long flags;668 int rc = 0;669 670 /* config types are set a boot time and never change */671 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&672 drvdata->config_type != TMC_CONFIG_TYPE_ETF))673 return -EINVAL;674 675 spin_lock_irqsave(&drvdata->spinlock, flags);676 677 /* Re-enable the TMC if need be */678 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {679 /* There is no point in reading a TMC in HW FIFO mode */680 mode = readl_relaxed(drvdata->base + TMC_MODE);681 if (mode != TMC_MODE_CIRCULAR_BUFFER) {682 spin_unlock_irqrestore(&drvdata->spinlock, flags);683 return -EINVAL;684 }685 /*686 * The trace run will continue with the same allocated trace687 * buffer. As such zero-out the buffer so that we don't end688 * up with stale data.689 *690 * Since the tracer is still enabled drvdata::buf691 * can't be NULL.692 */693 memset(drvdata->buf, 0, drvdata->size);694 rc = __tmc_etb_enable_hw(drvdata);695 if (rc) {696 spin_unlock_irqrestore(&drvdata->spinlock, flags);697 return rc;698 }699 } else {700 /*701 * The ETB/ETF is not tracing and the buffer was just read.702 * As such prepare to free the trace buffer.703 */704 buf = drvdata->buf;705 drvdata->buf = NULL;706 }707 708 drvdata->reading = false;709 spin_unlock_irqrestore(&drvdata->spinlock, flags);710 711 /*712 * Free allocated memory outside of the spinlock. There is no need713 * to assert the validity of 'buf' since calling kfree(NULL) is safe.714 */715 kfree(buf);716 717 return 0;718}719