1896 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright(C) 2016 Linaro Limited. All rights reserved.4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>5 */6 7#include <linux/atomic.h>8#include <linux/coresight.h>9#include <linux/dma-mapping.h>10#include <linux/iommu.h>11#include <linux/idr.h>12#include <linux/mutex.h>13#include <linux/refcount.h>14#include <linux/slab.h>15#include <linux/types.h>16#include <linux/vmalloc.h>17#include "coresight-catu.h"18#include "coresight-etm-perf.h"19#include "coresight-priv.h"20#include "coresight-tmc.h"21 22struct etr_flat_buf {23 struct device *dev;24 dma_addr_t daddr;25 void *vaddr;26 size_t size;27};28 29struct etr_buf_hw {30 bool has_iommu;31 bool has_etr_sg;32 bool has_catu;33};34 35/*36 * etr_perf_buffer - Perf buffer used for ETR37 * @drvdata - The ETR drvdaga this buffer has been allocated for.38 * @etr_buf - Actual buffer used by the ETR39 * @pid - The PID of the session owner that etr_perf_buffer40 * belongs to.41 * @snaphost - Perf session mode42 * @nr_pages - Number of pages in the ring buffer.43 * @pages - Array of Pages in the ring buffer.44 */45struct etr_perf_buffer {46 struct tmc_drvdata *drvdata;47 struct etr_buf *etr_buf;48 pid_t pid;49 bool snapshot;50 int nr_pages;51 void **pages;52};53 54/* Convert the perf index to an offset within the ETR buffer */55#define PERF_IDX2OFF(idx, buf) \56 ((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))57 58/* Lower limit for ETR hardware buffer */59#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M60 61/*62 * The TMC ETR SG has a page size of 4K. The SG table contains pointers63 * to 4KB buffers. However, the OS may use a PAGE_SIZE different from64 * 4K (i.e, 16KB or 64KB). This implies that a single OS page could65 * contain more than one SG buffer and tables.66 *67 * A table entry has the following format:68 *69 * ---Bit31------------Bit4-------Bit1-----Bit0--70 * | Address[39:12] | SBZ | Entry Type |71 * ----------------------------------------------72 *73 * Address: Bits [39:12] of a physical page address. Bits [11:0] are74 * always zero.75 *76 * Entry type:77 * b00 - Reserved.78 * b01 - Last entry in the tables, points to 4K page buffer.79 * b10 - Normal entry, points to 4K page buffer.80 * b11 - Link. The address points to the base of next table.81 */82 83typedef u32 sgte_t;84 85#define ETR_SG_PAGE_SHIFT 1286#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)87#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)88#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))89#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))90 91#define ETR_SG_ET_MASK 0x392#define ETR_SG_ET_LAST 0x193#define ETR_SG_ET_NORMAL 0x294#define ETR_SG_ET_LINK 0x395 96#define ETR_SG_ADDR_SHIFT 497 98#define ETR_SG_ENTRY(addr, type) \99 (sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \100 (type & ETR_SG_ET_MASK))101 102#define ETR_SG_ADDR(entry) \103 (((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)104#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)105 106/*107 * struct etr_sg_table : ETR SG Table108 * @sg_table: Generic SG Table holding the data/table pages.109 * @hwaddr: hwaddress used by the TMC, which is the base110 * address of the table.111 */112struct etr_sg_table {113 struct tmc_sg_table *sg_table;114 dma_addr_t hwaddr;115};116 117/*118 * tmc_etr_sg_table_entries: Total number of table entries required to map119 * @nr_pages system pages.120 *121 * We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.122 * Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,123 * with the last entry pointing to another page of table entries.124 * If we spill over to a new page for mapping 1 entry, we could as125 * well replace the link entry of the previous page with the last entry.126 */127static inline unsigned long __attribute_const__128tmc_etr_sg_table_entries(int nr_pages)129{130 unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;131 unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);132 /*133 * If we spill over to a new page for 1 entry, we could as well134 * make it the LAST entry in the previous page, skipping the Link135 * address.136 */137 if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))138 nr_sglinks--;139 return nr_sgpages + nr_sglinks;140}141 142/*143 * tmc_pages_get_offset: Go through all the pages in the tmc_pages144 * and map the device address @addr to an offset within the virtual145 * contiguous buffer.146 */147static long148tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)149{150 int i;151 dma_addr_t page_start;152 153 for (i = 0; i < tmc_pages->nr_pages; i++) {154 page_start = tmc_pages->daddrs[i];155 if (addr >= page_start && addr < (page_start + PAGE_SIZE))156 return i * PAGE_SIZE + (addr - page_start);157 }158 159 return -EINVAL;160}161 162/*163 * tmc_pages_free : Unmap and free the pages used by tmc_pages.164 * If the pages were not allocated in tmc_pages_alloc(), we would165 * simply drop the refcount.166 */167static void tmc_pages_free(struct tmc_pages *tmc_pages,168 struct device *dev, enum dma_data_direction dir)169{170 int i;171 struct device *real_dev = dev->parent;172 173 for (i = 0; i < tmc_pages->nr_pages; i++) {174 if (tmc_pages->daddrs && tmc_pages->daddrs[i])175 dma_unmap_page(real_dev, tmc_pages->daddrs[i],176 PAGE_SIZE, dir);177 if (tmc_pages->pages && tmc_pages->pages[i])178 __free_page(tmc_pages->pages[i]);179 }180 181 kfree(tmc_pages->pages);182 kfree(tmc_pages->daddrs);183 tmc_pages->pages = NULL;184 tmc_pages->daddrs = NULL;185 tmc_pages->nr_pages = 0;186}187 188/*189 * tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.190 * If @pages is not NULL, the list of page virtual addresses are191 * used as the data pages. The pages are then dma_map'ed for @dev192 * with dma_direction @dir.193 *194 * Returns 0 upon success, else the error number.195 */196static int tmc_pages_alloc(struct tmc_pages *tmc_pages,197 struct device *dev, int node,198 enum dma_data_direction dir, void **pages)199{200 int i, nr_pages;201 dma_addr_t paddr;202 struct page *page;203 struct device *real_dev = dev->parent;204 205 nr_pages = tmc_pages->nr_pages;206 tmc_pages->daddrs = kcalloc(nr_pages, sizeof(*tmc_pages->daddrs),207 GFP_KERNEL);208 if (!tmc_pages->daddrs)209 return -ENOMEM;210 tmc_pages->pages = kcalloc(nr_pages, sizeof(*tmc_pages->pages),211 GFP_KERNEL);212 if (!tmc_pages->pages) {213 kfree(tmc_pages->daddrs);214 tmc_pages->daddrs = NULL;215 return -ENOMEM;216 }217 218 for (i = 0; i < nr_pages; i++) {219 if (pages && pages[i]) {220 page = virt_to_page(pages[i]);221 /* Hold a refcount on the page */222 get_page(page);223 } else {224 page = alloc_pages_node(node,225 GFP_KERNEL | __GFP_ZERO, 0);226 if (!page)227 goto err;228 }229 paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);230 if (dma_mapping_error(real_dev, paddr))231 goto err;232 tmc_pages->daddrs[i] = paddr;233 tmc_pages->pages[i] = page;234 }235 return 0;236err:237 tmc_pages_free(tmc_pages, dev, dir);238 return -ENOMEM;239}240 241static inline long242tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)243{244 return tmc_pages_get_offset(&sg_table->data_pages, addr);245}246 247static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)248{249 if (sg_table->table_vaddr)250 vunmap(sg_table->table_vaddr);251 tmc_pages_free(&sg_table->table_pages, sg_table->dev, DMA_TO_DEVICE);252}253 254static void tmc_free_data_pages(struct tmc_sg_table *sg_table)255{256 if (sg_table->data_vaddr)257 vunmap(sg_table->data_vaddr);258 tmc_pages_free(&sg_table->data_pages, sg_table->dev, DMA_FROM_DEVICE);259}260 261void tmc_free_sg_table(struct tmc_sg_table *sg_table)262{263 tmc_free_table_pages(sg_table);264 tmc_free_data_pages(sg_table);265 kfree(sg_table);266}267EXPORT_SYMBOL_GPL(tmc_free_sg_table);268 269/*270 * Alloc pages for the table. Since this will be used by the device,271 * allocate the pages closer to the device (i.e, dev_to_node(dev)272 * rather than the CPU node).273 */274static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)275{276 int rc;277 struct tmc_pages *table_pages = &sg_table->table_pages;278 279 rc = tmc_pages_alloc(table_pages, sg_table->dev,280 dev_to_node(sg_table->dev),281 DMA_TO_DEVICE, NULL);282 if (rc)283 return rc;284 sg_table->table_vaddr = vmap(table_pages->pages,285 table_pages->nr_pages,286 VM_MAP,287 PAGE_KERNEL);288 if (!sg_table->table_vaddr)289 rc = -ENOMEM;290 else291 sg_table->table_daddr = table_pages->daddrs[0];292 return rc;293}294 295static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)296{297 int rc;298 299 /* Allocate data pages on the node requested by the caller */300 rc = tmc_pages_alloc(&sg_table->data_pages,301 sg_table->dev, sg_table->node,302 DMA_FROM_DEVICE, pages);303 if (!rc) {304 sg_table->data_vaddr = vmap(sg_table->data_pages.pages,305 sg_table->data_pages.nr_pages,306 VM_MAP,307 PAGE_KERNEL);308 if (!sg_table->data_vaddr)309 rc = -ENOMEM;310 }311 return rc;312}313 314/*315 * tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table316 * and data buffers. TMC writes to the data buffers and reads from the SG317 * Table pages.318 *319 * @dev - Coresight device to which page should be DMA mapped.320 * @node - Numa node for mem allocations321 * @nr_tpages - Number of pages for the table entries.322 * @nr_dpages - Number of pages for Data buffer.323 * @pages - Optional list of virtual address of pages.324 */325struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,326 int node,327 int nr_tpages,328 int nr_dpages,329 void **pages)330{331 long rc;332 struct tmc_sg_table *sg_table;333 334 sg_table = kzalloc(sizeof(*sg_table), GFP_KERNEL);335 if (!sg_table)336 return ERR_PTR(-ENOMEM);337 sg_table->data_pages.nr_pages = nr_dpages;338 sg_table->table_pages.nr_pages = nr_tpages;339 sg_table->node = node;340 sg_table->dev = dev;341 342 rc = tmc_alloc_data_pages(sg_table, pages);343 if (!rc)344 rc = tmc_alloc_table_pages(sg_table);345 if (rc) {346 tmc_free_sg_table(sg_table);347 return ERR_PTR(rc);348 }349 350 return sg_table;351}352EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);353 354/*355 * tmc_sg_table_sync_data_range: Sync the data buffer written356 * by the device from @offset upto a @size bytes.357 */358void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,359 u64 offset, u64 size)360{361 int i, index, start;362 int npages = DIV_ROUND_UP(size, PAGE_SIZE);363 struct device *real_dev = table->dev->parent;364 struct tmc_pages *data = &table->data_pages;365 366 start = offset >> PAGE_SHIFT;367 for (i = start; i < (start + npages); i++) {368 index = i % data->nr_pages;369 dma_sync_single_for_cpu(real_dev, data->daddrs[index],370 PAGE_SIZE, DMA_FROM_DEVICE);371 }372}373EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);374 375/* tmc_sg_sync_table: Sync the page table */376void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)377{378 int i;379 struct device *real_dev = sg_table->dev->parent;380 struct tmc_pages *table_pages = &sg_table->table_pages;381 382 for (i = 0; i < table_pages->nr_pages; i++)383 dma_sync_single_for_device(real_dev, table_pages->daddrs[i],384 PAGE_SIZE, DMA_TO_DEVICE);385}386EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);387 388/*389 * tmc_sg_table_get_data: Get the buffer pointer for data @offset390 * in the SG buffer. The @bufpp is updated to point to the buffer.391 * Returns :392 * the length of linear data available at @offset.393 * or394 * <= 0 if no data is available.395 */396ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,397 u64 offset, size_t len, char **bufpp)398{399 size_t size;400 int pg_idx = offset >> PAGE_SHIFT;401 int pg_offset = offset & (PAGE_SIZE - 1);402 struct tmc_pages *data_pages = &sg_table->data_pages;403 404 size = tmc_sg_table_buf_size(sg_table);405 if (offset >= size)406 return -EINVAL;407 408 /* Make sure we don't go beyond the end */409 len = (len < (size - offset)) ? len : size - offset;410 /* Respect the page boundaries */411 len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);412 if (len > 0)413 *bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;414 return len;415}416EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);417 418#ifdef ETR_SG_DEBUG419/* Map a dma address to virtual address */420static unsigned long421tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,422 dma_addr_t addr, bool table)423{424 long offset;425 unsigned long base;426 struct tmc_pages *tmc_pages;427 428 if (table) {429 tmc_pages = &sg_table->table_pages;430 base = (unsigned long)sg_table->table_vaddr;431 } else {432 tmc_pages = &sg_table->data_pages;433 base = (unsigned long)sg_table->data_vaddr;434 }435 436 offset = tmc_pages_get_offset(tmc_pages, addr);437 if (offset < 0)438 return 0;439 return base + offset;440}441 442/* Dump the given sg_table */443static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)444{445 sgte_t *ptr;446 int i = 0;447 dma_addr_t addr;448 struct tmc_sg_table *sg_table = etr_table->sg_table;449 450 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,451 etr_table->hwaddr, true);452 while (ptr) {453 addr = ETR_SG_ADDR(*ptr);454 switch (ETR_SG_ET(*ptr)) {455 case ETR_SG_ET_NORMAL:456 dev_dbg(sg_table->dev,457 "%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);458 ptr++;459 break;460 case ETR_SG_ET_LINK:461 dev_dbg(sg_table->dev,462 "%05d: *** %p\t:{L} 0x%llx ***\n",463 i, ptr, addr);464 ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,465 addr, true);466 break;467 case ETR_SG_ET_LAST:468 dev_dbg(sg_table->dev,469 "%05d: ### %p\t:[L] 0x%llx ###\n",470 i, ptr, addr);471 return;472 default:473 dev_dbg(sg_table->dev,474 "%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",475 i, ptr, addr);476 return;477 }478 i++;479 }480 dev_dbg(sg_table->dev, "******* End of Table *****\n");481}482#else483static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}484#endif485 486/*487 * Populate the SG Table page table entries from table/data488 * pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.489 * So does a Table page. So we keep track of indices of the tables490 * in each system page and move the pointers accordingly.491 */492#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))493static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)494{495 dma_addr_t paddr;496 int i, type, nr_entries;497 int tpidx = 0; /* index to the current system table_page */498 int sgtidx = 0; /* index to the sg_table within the current syspage */499 int sgtentry = 0; /* the entry within the sg_table */500 int dpidx = 0; /* index to the current system data_page */501 int spidx = 0; /* index to the SG page within the current data page */502 sgte_t *ptr; /* pointer to the table entry to fill */503 struct tmc_sg_table *sg_table = etr_table->sg_table;504 dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;505 dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;506 507 nr_entries = tmc_etr_sg_table_entries(sg_table->data_pages.nr_pages);508 /*509 * Use the contiguous virtual address of the table to update entries.510 */511 ptr = sg_table->table_vaddr;512 /*513 * Fill all the entries, except the last entry to avoid special514 * checks within the loop.515 */516 for (i = 0; i < nr_entries - 1; i++) {517 if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {518 /*519 * Last entry in a sg_table page is a link address to520 * the next table page. If this sg_table is the last521 * one in the system page, it links to the first522 * sg_table in the next system page. Otherwise, it523 * links to the next sg_table page within the system524 * page.525 */526 if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {527 paddr = table_daddrs[tpidx + 1];528 } else {529 paddr = table_daddrs[tpidx] +530 (ETR_SG_PAGE_SIZE * (sgtidx + 1));531 }532 type = ETR_SG_ET_LINK;533 } else {534 /*535 * Update the indices to the data_pages to point to the536 * next sg_page in the data buffer.537 */538 type = ETR_SG_ET_NORMAL;539 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;540 if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))541 dpidx++;542 }543 *ptr++ = ETR_SG_ENTRY(paddr, type);544 /*545 * Move to the next table pointer, moving the table page index546 * if necessary547 */548 if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {549 if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))550 tpidx++;551 }552 }553 554 /* Set up the last entry, which is always a data pointer */555 paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;556 *ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);557}558 559/*560 * tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and561 * populate the table.562 *563 * @dev - Device pointer for the TMC564 * @node - NUMA node where the memory should be allocated565 * @size - Total size of the data buffer566 * @pages - Optional list of page virtual address567 */568static struct etr_sg_table *569tmc_init_etr_sg_table(struct device *dev, int node,570 unsigned long size, void **pages)571{572 int nr_entries, nr_tpages;573 int nr_dpages = size >> PAGE_SHIFT;574 struct tmc_sg_table *sg_table;575 struct etr_sg_table *etr_table;576 577 etr_table = kzalloc(sizeof(*etr_table), GFP_KERNEL);578 if (!etr_table)579 return ERR_PTR(-ENOMEM);580 nr_entries = tmc_etr_sg_table_entries(nr_dpages);581 nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);582 583 sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);584 if (IS_ERR(sg_table)) {585 kfree(etr_table);586 return ERR_CAST(sg_table);587 }588 589 etr_table->sg_table = sg_table;590 /* TMC should use table base address for DBA */591 etr_table->hwaddr = sg_table->table_daddr;592 tmc_etr_sg_table_populate(etr_table);593 /* Sync the table pages for the HW */594 tmc_sg_table_sync_table(sg_table);595 tmc_etr_sg_table_dump(etr_table);596 597 return etr_table;598}599 600/*601 * tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.602 */603static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,604 struct etr_buf *etr_buf, int node,605 void **pages)606{607 struct etr_flat_buf *flat_buf;608 struct device *real_dev = drvdata->csdev->dev.parent;609 610 /* We cannot reuse existing pages for flat buf */611 if (pages)612 return -EINVAL;613 614 flat_buf = kzalloc(sizeof(*flat_buf), GFP_KERNEL);615 if (!flat_buf)616 return -ENOMEM;617 618 flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size,619 &flat_buf->daddr,620 DMA_FROM_DEVICE,621 GFP_KERNEL | __GFP_NOWARN);622 if (!flat_buf->vaddr) {623 kfree(flat_buf);624 return -ENOMEM;625 }626 627 flat_buf->size = etr_buf->size;628 flat_buf->dev = &drvdata->csdev->dev;629 etr_buf->hwaddr = flat_buf->daddr;630 etr_buf->mode = ETR_MODE_FLAT;631 etr_buf->private = flat_buf;632 return 0;633}634 635static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)636{637 struct etr_flat_buf *flat_buf = etr_buf->private;638 639 if (flat_buf && flat_buf->daddr) {640 struct device *real_dev = flat_buf->dev->parent;641 642 dma_free_noncoherent(real_dev, etr_buf->size,643 flat_buf->vaddr, flat_buf->daddr,644 DMA_FROM_DEVICE);645 }646 kfree(flat_buf);647}648 649static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)650{651 struct etr_flat_buf *flat_buf = etr_buf->private;652 struct device *real_dev = flat_buf->dev->parent;653 654 /*655 * Adjust the buffer to point to the beginning of the trace data656 * and update the available trace data.657 */658 etr_buf->offset = rrp - etr_buf->hwaddr;659 if (etr_buf->full)660 etr_buf->len = etr_buf->size;661 else662 etr_buf->len = rwp - rrp;663 664 /*665 * The driver always starts tracing at the beginning of the buffer,666 * the only reason why we would get a wrap around is when the buffer667 * is full. Sync the entire buffer in one go for this case.668 */669 if (etr_buf->offset + etr_buf->len > etr_buf->size)670 dma_sync_single_for_cpu(real_dev, flat_buf->daddr,671 etr_buf->size, DMA_FROM_DEVICE);672 else673 dma_sync_single_for_cpu(real_dev,674 flat_buf->daddr + etr_buf->offset,675 etr_buf->len, DMA_FROM_DEVICE);676}677 678static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,679 u64 offset, size_t len, char **bufpp)680{681 struct etr_flat_buf *flat_buf = etr_buf->private;682 683 *bufpp = (char *)flat_buf->vaddr + offset;684 /*685 * tmc_etr_buf_get_data already adjusts the length to handle686 * buffer wrapping around.687 */688 return len;689}690 691static const struct etr_buf_operations etr_flat_buf_ops = {692 .alloc = tmc_etr_alloc_flat_buf,693 .free = tmc_etr_free_flat_buf,694 .sync = tmc_etr_sync_flat_buf,695 .get_data = tmc_etr_get_data_flat_buf,696};697 698/*699 * tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters700 * appropriately.701 */702static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,703 struct etr_buf *etr_buf, int node,704 void **pages)705{706 struct etr_sg_table *etr_table;707 struct device *dev = &drvdata->csdev->dev;708 709 etr_table = tmc_init_etr_sg_table(dev, node,710 etr_buf->size, pages);711 if (IS_ERR(etr_table))712 return -ENOMEM;713 etr_buf->hwaddr = etr_table->hwaddr;714 etr_buf->mode = ETR_MODE_ETR_SG;715 etr_buf->private = etr_table;716 return 0;717}718 719static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)720{721 struct etr_sg_table *etr_table = etr_buf->private;722 723 if (etr_table) {724 tmc_free_sg_table(etr_table->sg_table);725 kfree(etr_table);726 }727}728 729static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,730 size_t len, char **bufpp)731{732 struct etr_sg_table *etr_table = etr_buf->private;733 734 return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);735}736 737static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)738{739 long r_offset, w_offset;740 struct etr_sg_table *etr_table = etr_buf->private;741 struct tmc_sg_table *table = etr_table->sg_table;742 743 /* Convert hw address to offset in the buffer */744 r_offset = tmc_sg_get_data_page_offset(table, rrp);745 if (r_offset < 0) {746 dev_warn(table->dev,747 "Unable to map RRP %llx to offset\n", rrp);748 etr_buf->len = 0;749 return;750 }751 752 w_offset = tmc_sg_get_data_page_offset(table, rwp);753 if (w_offset < 0) {754 dev_warn(table->dev,755 "Unable to map RWP %llx to offset\n", rwp);756 etr_buf->len = 0;757 return;758 }759 760 etr_buf->offset = r_offset;761 if (etr_buf->full)762 etr_buf->len = etr_buf->size;763 else764 etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +765 w_offset - r_offset;766 tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);767}768 769static const struct etr_buf_operations etr_sg_buf_ops = {770 .alloc = tmc_etr_alloc_sg_buf,771 .free = tmc_etr_free_sg_buf,772 .sync = tmc_etr_sync_sg_buf,773 .get_data = tmc_etr_get_data_sg_buf,774};775 776/*777 * TMC ETR could be connected to a CATU device, which can provide address778 * translation service. This is represented by the Output port of the TMC779 * (ETR) connected to the input port of the CATU.780 *781 * Returns : coresight_device ptr for the CATU device if a CATU is found.782 * : NULL otherwise.783 */784struct coresight_device *785tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)786{787 struct coresight_device *etr = drvdata->csdev;788 union coresight_dev_subtype catu_subtype = {789 .helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU790 };791 792 if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))793 return NULL;794 795 return coresight_find_output_type(etr->pdata, CORESIGHT_DEV_TYPE_HELPER,796 catu_subtype);797}798EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);799 800static const struct etr_buf_operations *etr_buf_ops[] = {801 [ETR_MODE_FLAT] = &etr_flat_buf_ops,802 [ETR_MODE_ETR_SG] = &etr_sg_buf_ops,803 [ETR_MODE_CATU] = NULL,804};805 806void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)807{808 etr_buf_ops[ETR_MODE_CATU] = catu;809}810EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);811 812void tmc_etr_remove_catu_ops(void)813{814 etr_buf_ops[ETR_MODE_CATU] = NULL;815}816EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);817 818static inline int tmc_etr_mode_alloc_buf(int mode,819 struct tmc_drvdata *drvdata,820 struct etr_buf *etr_buf, int node,821 void **pages)822{823 int rc = -EINVAL;824 825 switch (mode) {826 case ETR_MODE_FLAT:827 case ETR_MODE_ETR_SG:828 case ETR_MODE_CATU:829 if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)830 rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,831 node, pages);832 if (!rc)833 etr_buf->ops = etr_buf_ops[mode];834 return rc;835 default:836 return -EINVAL;837 }838}839 840static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)841{842 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);843 844 buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);845 buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);846 buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);847}848 849static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)850{851 bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;852 853 return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;854}855 856/*857 * tmc_alloc_etr_buf: Allocate a buffer use by ETR.858 * @drvdata : ETR device details.859 * @size : size of the requested buffer.860 * @flags : Required properties for the buffer.861 * @node : Node for memory allocations.862 * @pages : An optional list of pages.863 */864static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,865 ssize_t size, int flags,866 int node, void **pages)867{868 int rc = -ENOMEM;869 struct etr_buf *etr_buf;870 struct etr_buf_hw buf_hw;871 struct device *dev = &drvdata->csdev->dev;872 873 get_etr_buf_hw(dev, &buf_hw);874 etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);875 if (!etr_buf)876 return ERR_PTR(-ENOMEM);877 878 etr_buf->size = size;879 880 /* If there is user directive for buffer mode, try that first */881 if (drvdata->etr_mode != ETR_MODE_AUTO)882 rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,883 etr_buf, node, pages);884 885 /*886 * If we have to use an existing list of pages, we cannot reliably887 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,888 * we use the contiguous DMA memory if at least one of the following889 * conditions is true:890 * a) The ETR cannot use Scatter-Gather.891 * b) we have a backing IOMMU892 * c) The requested memory size is smaller (< 1M).893 *894 * Fallback to available mechanisms.895 *896 */897 if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))898 rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,899 etr_buf, node, pages);900 if (rc && buf_hw.has_etr_sg)901 rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,902 etr_buf, node, pages);903 if (rc && buf_hw.has_catu)904 rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,905 etr_buf, node, pages);906 if (rc) {907 kfree(etr_buf);908 return ERR_PTR(rc);909 }910 911 refcount_set(&etr_buf->refcount, 1);912 dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",913 (unsigned long)size >> 10, etr_buf->mode);914 return etr_buf;915}916 917static void tmc_free_etr_buf(struct etr_buf *etr_buf)918{919 WARN_ON(!etr_buf->ops || !etr_buf->ops->free);920 etr_buf->ops->free(etr_buf);921 kfree(etr_buf);922}923 924/*925 * tmc_etr_buf_get_data: Get the pointer the trace data at @offset926 * with a maximum of @len bytes.927 * Returns: The size of the linear data available @pos, with *bufpp928 * updated to point to the buffer.929 */930static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,931 u64 offset, size_t len, char **bufpp)932{933 /* Adjust the length to limit this transaction to end of buffer */934 len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;935 936 return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);937}938 939static inline s64940tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)941{942 ssize_t len;943 char *bufp;944 945 len = tmc_etr_buf_get_data(etr_buf, offset,946 CORESIGHT_BARRIER_PKT_SIZE, &bufp);947 if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))948 return -EINVAL;949 coresight_insert_barrier_packet(bufp);950 return offset + CORESIGHT_BARRIER_PKT_SIZE;951}952 953/*954 * tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.955 * Makes sure the trace data is synced to the memory for consumption.956 * @etr_buf->offset will hold the offset to the beginning of the trace data957 * within the buffer, with @etr_buf->len bytes to consume.958 */959static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)960{961 struct etr_buf *etr_buf = drvdata->etr_buf;962 u64 rrp, rwp;963 u32 status;964 965 rrp = tmc_read_rrp(drvdata);966 rwp = tmc_read_rwp(drvdata);967 status = readl_relaxed(drvdata->base + TMC_STS);968 969 /*970 * If there were memory errors in the session, truncate the971 * buffer.972 */973 if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {974 dev_dbg(&drvdata->csdev->dev,975 "tmc memory error detected, truncating buffer\n");976 etr_buf->len = 0;977 etr_buf->full = false;978 return;979 }980 981 etr_buf->full = !!(status & TMC_STS_FULL);982 983 WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);984 985 etr_buf->ops->sync(etr_buf, rrp, rwp);986}987 988static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)989{990 u32 axictl, sts;991 struct etr_buf *etr_buf = drvdata->etr_buf;992 int rc = 0;993 994 CS_UNLOCK(drvdata->base);995 996 /* Wait for TMCSReady bit to be set */997 rc = tmc_wait_for_tmcready(drvdata);998 if (rc) {999 dev_err(&drvdata->csdev->dev,1000 "Failed to enable : TMC not ready\n");1001 CS_LOCK(drvdata->base);1002 return rc;1003 }1004 1005 writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);1006 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);1007 1008 axictl = readl_relaxed(drvdata->base + TMC_AXICTL);1009 axictl &= ~TMC_AXICTL_CLEAR_MASK;1010 axictl |= TMC_AXICTL_PROT_CTL_B1;1011 axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);1012 axictl |= TMC_AXICTL_AXCACHE_OS;1013 1014 if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {1015 axictl &= ~TMC_AXICTL_ARCACHE_MASK;1016 axictl |= TMC_AXICTL_ARCACHE_OS;1017 }1018 1019 if (etr_buf->mode == ETR_MODE_ETR_SG)1020 axictl |= TMC_AXICTL_SCT_GAT_MODE;1021 1022 writel_relaxed(axictl, drvdata->base + TMC_AXICTL);1023 tmc_write_dba(drvdata, etr_buf->hwaddr);1024 /*1025 * If the TMC pointers must be programmed before the session,1026 * we have to set it properly (i.e, RRP/RWP to base address and1027 * STS to "not full").1028 */1029 if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {1030 tmc_write_rrp(drvdata, etr_buf->hwaddr);1031 tmc_write_rwp(drvdata, etr_buf->hwaddr);1032 sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;1033 writel_relaxed(sts, drvdata->base + TMC_STS);1034 }1035 1036 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |1037 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |1038 TMC_FFCR_TRIGON_TRIGIN,1039 drvdata->base + TMC_FFCR);1040 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);1041 tmc_enable_hw(drvdata);1042 1043 CS_LOCK(drvdata->base);1044 return rc;1045}1046 1047static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,1048 struct etr_buf *etr_buf)1049{1050 int rc;1051 1052 /* Callers should provide an appropriate buffer for use */1053 if (WARN_ON(!etr_buf))1054 return -EINVAL;1055 1056 if ((etr_buf->mode == ETR_MODE_ETR_SG) &&1057 WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))1058 return -EINVAL;1059 1060 if (WARN_ON(drvdata->etr_buf))1061 return -EBUSY;1062 1063 rc = coresight_claim_device(drvdata->csdev);1064 if (!rc) {1065 drvdata->etr_buf = etr_buf;1066 rc = __tmc_etr_enable_hw(drvdata);1067 if (rc) {1068 drvdata->etr_buf = NULL;1069 coresight_disclaim_device(drvdata->csdev);1070 }1071 }1072 1073 return rc;1074}1075 1076/*1077 * Return the available trace data in the buffer (starts at etr_buf->offset,1078 * limited by etr_buf->len) from @pos, with a maximum limit of @len,1079 * also updating the @bufpp on where to find it. Since the trace data1080 * starts at anywhere in the buffer, depending on the RRP, we adjust the1081 * @len returned to handle buffer wrapping around.1082 *1083 * We are protected here by drvdata->reading != 0, which ensures the1084 * sysfs_buf stays alive.1085 */1086ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,1087 loff_t pos, size_t len, char **bufpp)1088{1089 s64 offset;1090 ssize_t actual = len;1091 struct etr_buf *etr_buf = drvdata->sysfs_buf;1092 1093 if (pos + actual > etr_buf->len)1094 actual = etr_buf->len - pos;1095 if (actual <= 0)1096 return actual;1097 1098 /* Compute the offset from which we read the data */1099 offset = etr_buf->offset + pos;1100 if (offset >= etr_buf->size)1101 offset -= etr_buf->size;1102 return tmc_etr_buf_get_data(etr_buf, offset, actual, bufpp);1103}1104 1105static struct etr_buf *1106tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)1107{1108 return tmc_alloc_etr_buf(drvdata, drvdata->size,1109 0, cpu_to_node(0), NULL);1110}1111 1112static void1113tmc_etr_free_sysfs_buf(struct etr_buf *buf)1114{1115 if (buf)1116 tmc_free_etr_buf(buf);1117}1118 1119static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)1120{1121 struct etr_buf *etr_buf = drvdata->etr_buf;1122 1123 if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {1124 tmc_etr_free_sysfs_buf(drvdata->sysfs_buf);1125 drvdata->sysfs_buf = NULL;1126 } else {1127 tmc_sync_etr_buf(drvdata);1128 /*1129 * Insert barrier packets at the beginning, if there was1130 * an overflow.1131 */1132 if (etr_buf->full)1133 tmc_etr_buf_insert_barrier_packet(etr_buf,1134 etr_buf->offset);1135 }1136}1137 1138static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)1139{1140 CS_UNLOCK(drvdata->base);1141 1142 tmc_flush_and_stop(drvdata);1143 /*1144 * When operating in sysFS mode the content of the buffer needs to be1145 * read before the TMC is disabled.1146 */1147 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)1148 tmc_etr_sync_sysfs_buf(drvdata);1149 1150 tmc_disable_hw(drvdata);1151 1152 CS_LOCK(drvdata->base);1153 1154}1155 1156void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)1157{1158 __tmc_etr_disable_hw(drvdata);1159 coresight_disclaim_device(drvdata->csdev);1160 /* Reset the ETR buf used by hardware */1161 drvdata->etr_buf = NULL;1162}1163 1164static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)1165{1166 int ret = 0;1167 unsigned long flags;1168 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1169 struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;1170 1171 /*1172 * If we are enabling the ETR from disabled state, we need to make1173 * sure we have a buffer with the right size. The etr_buf is not reset1174 * immediately after we stop the tracing in SYSFS mode as we wait for1175 * the user to collect the data. We may be able to reuse the existing1176 * buffer, provided the size matches. Any allocation has to be done1177 * with the lock released.1178 */1179 spin_lock_irqsave(&drvdata->spinlock, flags);1180 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);1181 if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {1182 spin_unlock_irqrestore(&drvdata->spinlock, flags);1183 1184 /* Allocate memory with the locks released */1185 free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);1186 if (IS_ERR(new_buf))1187 return new_buf;1188 1189 /* Let's try again */1190 spin_lock_irqsave(&drvdata->spinlock, flags);1191 }1192 1193 if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) {1194 ret = -EBUSY;1195 goto out;1196 }1197 1198 /*1199 * If we don't have a buffer or it doesn't match the requested size,1200 * use the buffer allocated above. Otherwise reuse the existing buffer.1201 */1202 sysfs_buf = READ_ONCE(drvdata->sysfs_buf);1203 if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {1204 free_buf = sysfs_buf;1205 drvdata->sysfs_buf = new_buf;1206 }1207 1208out:1209 spin_unlock_irqrestore(&drvdata->spinlock, flags);1210 1211 /* Free memory outside the spinlock if need be */1212 if (free_buf)1213 tmc_etr_free_sysfs_buf(free_buf);1214 return ret ? ERR_PTR(ret) : drvdata->sysfs_buf;1215}1216 1217static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)1218{1219 int ret = 0;1220 unsigned long flags;1221 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1222 struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);1223 1224 if (IS_ERR(sysfs_buf))1225 return PTR_ERR(sysfs_buf);1226 1227 spin_lock_irqsave(&drvdata->spinlock, flags);1228 1229 /*1230 * In sysFS mode we can have multiple writers per sink. Since this1231 * sink is already enabled no memory is needed and the HW need not be1232 * touched, even if the buffer size has changed.1233 */1234 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {1235 csdev->refcnt++;1236 goto out;1237 }1238 1239 ret = tmc_etr_enable_hw(drvdata, sysfs_buf);1240 if (!ret) {1241 coresight_set_mode(csdev, CS_MODE_SYSFS);1242 csdev->refcnt++;1243 }1244 1245out:1246 spin_unlock_irqrestore(&drvdata->spinlock, flags);1247 1248 if (!ret)1249 dev_dbg(&csdev->dev, "TMC-ETR enabled\n");1250 1251 return ret;1252}1253 1254struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,1255 enum cs_mode mode, void *data)1256{1257 struct perf_output_handle *handle = data;1258 struct etr_perf_buffer *etr_perf;1259 1260 switch (mode) {1261 case CS_MODE_SYSFS:1262 return tmc_etr_get_sysfs_buffer(csdev);1263 case CS_MODE_PERF:1264 etr_perf = etm_perf_sink_config(handle);1265 if (WARN_ON(!etr_perf || !etr_perf->etr_buf))1266 return ERR_PTR(-EINVAL);1267 return etr_perf->etr_buf;1268 default:1269 return ERR_PTR(-EINVAL);1270 }1271}1272EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);1273 1274/*1275 * alloc_etr_buf: Allocate ETR buffer for use by perf.1276 * The size of the hardware buffer is dependent on the size configured1277 * via sysfs and the perf ring buffer size. We prefer to allocate the1278 * largest possible size, scaling down the size by half until it1279 * reaches a minimum limit (1M), beyond which we give up.1280 */1281static struct etr_buf *1282alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,1283 int nr_pages, void **pages, bool snapshot)1284{1285 int node;1286 struct etr_buf *etr_buf;1287 unsigned long size;1288 1289 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);1290 /*1291 * Try to match the perf ring buffer size if it is larger1292 * than the size requested via sysfs.1293 */1294 if ((nr_pages << PAGE_SHIFT) > drvdata->size) {1295 etr_buf = tmc_alloc_etr_buf(drvdata, ((ssize_t)nr_pages << PAGE_SHIFT),1296 0, node, NULL);1297 if (!IS_ERR(etr_buf))1298 goto done;1299 }1300 1301 /*1302 * Else switch to configured size for this ETR1303 * and scale down until we hit the minimum limit.1304 */1305 size = drvdata->size;1306 do {1307 etr_buf = tmc_alloc_etr_buf(drvdata, size, 0, node, NULL);1308 if (!IS_ERR(etr_buf))1309 goto done;1310 size /= 2;1311 } while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);1312 1313 return ERR_PTR(-ENOMEM);1314 1315done:1316 return etr_buf;1317}1318 1319static struct etr_buf *1320get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,1321 struct perf_event *event, int nr_pages,1322 void **pages, bool snapshot)1323{1324 int ret;1325 pid_t pid = task_pid_nr(event->owner);1326 struct etr_buf *etr_buf;1327 1328retry:1329 /*1330 * An etr_perf_buffer is associated with an event and holds a reference1331 * to the AUX ring buffer that was created for that event. In CPU-wide1332 * N:1 mode multiple events (one per CPU), each with its own AUX ring1333 * buffer, share a sink. As such an etr_perf_buffer is created for each1334 * event but a single etr_buf associated with the ETR is shared between1335 * them. The last event in a trace session will copy the content of the1336 * etr_buf to its AUX ring buffer. Ring buffer associated to other1337 * events are simply not used an freed as events are destoyed. We still1338 * need to allocate a ring buffer for each event since we don't know1339 * which event will be last.1340 */1341 1342 /*1343 * The first thing to do here is check if an etr_buf has already been1344 * allocated for this session. If so it is shared with this event,1345 * otherwise it is created.1346 */1347 mutex_lock(&drvdata->idr_mutex);1348 etr_buf = idr_find(&drvdata->idr, pid);1349 if (etr_buf) {1350 refcount_inc(&etr_buf->refcount);1351 mutex_unlock(&drvdata->idr_mutex);1352 return etr_buf;1353 }1354 1355 /* If we made it here no buffer has been allocated, do so now. */1356 mutex_unlock(&drvdata->idr_mutex);1357 1358 etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);1359 if (IS_ERR(etr_buf))1360 return etr_buf;1361 1362 /* Now that we have a buffer, add it to the IDR. */1363 mutex_lock(&drvdata->idr_mutex);1364 ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL);1365 mutex_unlock(&drvdata->idr_mutex);1366 1367 /* Another event with this session ID has allocated this buffer. */1368 if (ret == -ENOSPC) {1369 tmc_free_etr_buf(etr_buf);1370 goto retry;1371 }1372 1373 /* The IDR can't allocate room for a new session, abandon ship. */1374 if (ret == -ENOMEM) {1375 tmc_free_etr_buf(etr_buf);1376 return ERR_PTR(ret);1377 }1378 1379 1380 return etr_buf;1381}1382 1383static struct etr_buf *1384get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,1385 struct perf_event *event, int nr_pages,1386 void **pages, bool snapshot)1387{1388 /*1389 * In per-thread mode the etr_buf isn't shared, so just go ahead1390 * with memory allocation.1391 */1392 return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);1393}1394 1395static struct etr_buf *1396get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,1397 int nr_pages, void **pages, bool snapshot)1398{1399 if (event->cpu == -1)1400 return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,1401 pages, snapshot);1402 1403 return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,1404 pages, snapshot);1405}1406 1407static struct etr_perf_buffer *1408tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,1409 int nr_pages, void **pages, bool snapshot)1410{1411 int node;1412 struct etr_buf *etr_buf;1413 struct etr_perf_buffer *etr_perf;1414 1415 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);1416 1417 etr_perf = kzalloc_node(sizeof(*etr_perf), GFP_KERNEL, node);1418 if (!etr_perf)1419 return ERR_PTR(-ENOMEM);1420 1421 etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);1422 if (!IS_ERR(etr_buf))1423 goto done;1424 1425 kfree(etr_perf);1426 return ERR_PTR(-ENOMEM);1427 1428done:1429 /*1430 * Keep a reference to the ETR this buffer has been allocated for1431 * in order to have access to the IDR in tmc_free_etr_buffer().1432 */1433 etr_perf->drvdata = drvdata;1434 etr_perf->etr_buf = etr_buf;1435 1436 return etr_perf;1437}1438 1439 1440static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,1441 struct perf_event *event, void **pages,1442 int nr_pages, bool snapshot)1443{1444 struct etr_perf_buffer *etr_perf;1445 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1446 1447 etr_perf = tmc_etr_setup_perf_buf(drvdata, event,1448 nr_pages, pages, snapshot);1449 if (IS_ERR(etr_perf)) {1450 dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");1451 return NULL;1452 }1453 1454 etr_perf->pid = task_pid_nr(event->owner);1455 etr_perf->snapshot = snapshot;1456 etr_perf->nr_pages = nr_pages;1457 etr_perf->pages = pages;1458 1459 return etr_perf;1460}1461 1462static void tmc_free_etr_buffer(void *config)1463{1464 struct etr_perf_buffer *etr_perf = config;1465 struct tmc_drvdata *drvdata = etr_perf->drvdata;1466 struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;1467 1468 if (!etr_buf)1469 goto free_etr_perf_buffer;1470 1471 mutex_lock(&drvdata->idr_mutex);1472 /* If we are not the last one to use the buffer, don't touch it. */1473 if (!refcount_dec_and_test(&etr_buf->refcount)) {1474 mutex_unlock(&drvdata->idr_mutex);1475 goto free_etr_perf_buffer;1476 }1477 1478 /* We are the last one, remove from the IDR and free the buffer. */1479 buf = idr_remove(&drvdata->idr, etr_perf->pid);1480 mutex_unlock(&drvdata->idr_mutex);1481 1482 /*1483 * Something went very wrong if the buffer associated with this ID1484 * is not the same in the IDR. Leak to avoid use after free.1485 */1486 if (buf && WARN_ON(buf != etr_buf))1487 goto free_etr_perf_buffer;1488 1489 tmc_free_etr_buf(etr_perf->etr_buf);1490 1491free_etr_perf_buffer:1492 kfree(etr_perf);1493}1494 1495/*1496 * tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware1497 * buffer to the perf ring buffer.1498 */1499static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,1500 unsigned long head,1501 unsigned long src_offset,1502 unsigned long to_copy)1503{1504 long bytes;1505 long pg_idx, pg_offset;1506 char **dst_pages, *src_buf;1507 struct etr_buf *etr_buf = etr_perf->etr_buf;1508 1509 head = PERF_IDX2OFF(head, etr_perf);1510 pg_idx = head >> PAGE_SHIFT;1511 pg_offset = head & (PAGE_SIZE - 1);1512 dst_pages = (char **)etr_perf->pages;1513 1514 while (to_copy > 0) {1515 /*1516 * In one iteration, we can copy minimum of :1517 * 1) what is available in the source buffer,1518 * 2) what is available in the source buffer, before it1519 * wraps around.1520 * 3) what is available in the destination page.1521 * in one iteration.1522 */1523 if (src_offset >= etr_buf->size)1524 src_offset -= etr_buf->size;1525 bytes = tmc_etr_buf_get_data(etr_buf, src_offset, to_copy,1526 &src_buf);1527 if (WARN_ON_ONCE(bytes <= 0))1528 break;1529 bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));1530 1531 memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);1532 1533 to_copy -= bytes;1534 1535 /* Move destination pointers */1536 pg_offset += bytes;1537 if (pg_offset == PAGE_SIZE) {1538 pg_offset = 0;1539 if (++pg_idx == etr_perf->nr_pages)1540 pg_idx = 0;1541 }1542 1543 /* Move source pointers */1544 src_offset += bytes;1545 }1546}1547 1548/*1549 * tmc_update_etr_buffer : Update the perf ring buffer with the1550 * available trace data. We use software double buffering at the moment.1551 *1552 * TODO: Add support for reusing the perf ring buffer.1553 */1554static unsigned long1555tmc_update_etr_buffer(struct coresight_device *csdev,1556 struct perf_output_handle *handle,1557 void *config)1558{1559 bool lost = false;1560 unsigned long flags, offset, size = 0;1561 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1562 struct etr_perf_buffer *etr_perf = config;1563 struct etr_buf *etr_buf = etr_perf->etr_buf;1564 1565 spin_lock_irqsave(&drvdata->spinlock, flags);1566 1567 /* Don't do anything if another tracer is using this sink */1568 if (csdev->refcnt != 1) {1569 spin_unlock_irqrestore(&drvdata->spinlock, flags);1570 goto out;1571 }1572 1573 if (WARN_ON(drvdata->perf_buf != etr_buf)) {1574 lost = true;1575 spin_unlock_irqrestore(&drvdata->spinlock, flags);1576 goto out;1577 }1578 1579 CS_UNLOCK(drvdata->base);1580 1581 tmc_flush_and_stop(drvdata);1582 tmc_sync_etr_buf(drvdata);1583 1584 CS_LOCK(drvdata->base);1585 spin_unlock_irqrestore(&drvdata->spinlock, flags);1586 1587 lost = etr_buf->full;1588 offset = etr_buf->offset;1589 size = etr_buf->len;1590 1591 /*1592 * The ETR buffer may be bigger than the space available in the1593 * perf ring buffer (handle->size). If so advance the offset so that we1594 * get the latest trace data. In snapshot mode none of that matters1595 * since we are expected to clobber stale data in favour of the latest1596 * traces.1597 */1598 if (!etr_perf->snapshot && size > handle->size) {1599 u32 mask = tmc_get_memwidth_mask(drvdata);1600 1601 /*1602 * Make sure the new size is aligned in accordance with the1603 * requirement explained in function tmc_get_memwidth_mask().1604 */1605 size = handle->size & mask;1606 offset = etr_buf->offset + etr_buf->len - size;1607 1608 if (offset >= etr_buf->size)1609 offset -= etr_buf->size;1610 lost = true;1611 }1612 1613 /* Insert barrier packets at the beginning, if there was an overflow */1614 if (lost)1615 tmc_etr_buf_insert_barrier_packet(etr_buf, offset);1616 tmc_etr_sync_perf_buffer(etr_perf, handle->head, offset, size);1617 1618 /*1619 * In snapshot mode we simply increment the head by the number of byte1620 * that were written. User space will figure out how many bytes to get1621 * from the AUX buffer based on the position of the head.1622 */1623 if (etr_perf->snapshot)1624 handle->head += size;1625 1626 /*1627 * Ensure that the AUX trace data is visible before the aux_head1628 * is updated via perf_aux_output_end(), as expected by the1629 * perf ring buffer.1630 */1631 smp_wmb();1632 1633out:1634 /*1635 * Don't set the TRUNCATED flag in snapshot mode because 1) the1636 * captured buffer is expected to be truncated and 2) a full buffer1637 * prevents the event from being re-enabled by the perf core,1638 * resulting in stale data being send to user space.1639 */1640 if (!etr_perf->snapshot && lost)1641 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);1642 return size;1643}1644 1645static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)1646{1647 int rc = 0;1648 pid_t pid;1649 unsigned long flags;1650 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1651 struct perf_output_handle *handle = data;1652 struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);1653 1654 spin_lock_irqsave(&drvdata->spinlock, flags);1655 /* Don't use this sink if it is already claimed by sysFS */1656 if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {1657 rc = -EBUSY;1658 goto unlock_out;1659 }1660 1661 if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {1662 rc = -EINVAL;1663 goto unlock_out;1664 }1665 1666 /* Get a handle on the pid of the session owner */1667 pid = etr_perf->pid;1668 1669 /* Do not proceed if this device is associated with another session */1670 if (drvdata->pid != -1 && drvdata->pid != pid) {1671 rc = -EBUSY;1672 goto unlock_out;1673 }1674 1675 /*1676 * No HW configuration is needed if the sink is already in1677 * use for this session.1678 */1679 if (drvdata->pid == pid) {1680 csdev->refcnt++;1681 goto unlock_out;1682 }1683 1684 rc = tmc_etr_enable_hw(drvdata, etr_perf->etr_buf);1685 if (!rc) {1686 /* Associate with monitored process. */1687 drvdata->pid = pid;1688 coresight_set_mode(csdev, CS_MODE_PERF);1689 drvdata->perf_buf = etr_perf->etr_buf;1690 csdev->refcnt++;1691 }1692 1693unlock_out:1694 spin_unlock_irqrestore(&drvdata->spinlock, flags);1695 return rc;1696}1697 1698static int tmc_enable_etr_sink(struct coresight_device *csdev,1699 enum cs_mode mode, void *data)1700{1701 switch (mode) {1702 case CS_MODE_SYSFS:1703 return tmc_enable_etr_sink_sysfs(csdev);1704 case CS_MODE_PERF:1705 return tmc_enable_etr_sink_perf(csdev, data);1706 default:1707 return -EINVAL;1708 }1709}1710 1711static int tmc_disable_etr_sink(struct coresight_device *csdev)1712{1713 unsigned long flags;1714 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);1715 1716 spin_lock_irqsave(&drvdata->spinlock, flags);1717 1718 if (drvdata->reading) {1719 spin_unlock_irqrestore(&drvdata->spinlock, flags);1720 return -EBUSY;1721 }1722 1723 csdev->refcnt--;1724 if (csdev->refcnt) {1725 spin_unlock_irqrestore(&drvdata->spinlock, flags);1726 return -EBUSY;1727 }1728 1729 /* Complain if we (somehow) got out of sync */1730 WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);1731 tmc_etr_disable_hw(drvdata);1732 /* Dissociate from monitored process. */1733 drvdata->pid = -1;1734 coresight_set_mode(csdev, CS_MODE_DISABLED);1735 /* Reset perf specific data */1736 drvdata->perf_buf = NULL;1737 1738 spin_unlock_irqrestore(&drvdata->spinlock, flags);1739 1740 dev_dbg(&csdev->dev, "TMC-ETR disabled\n");1741 return 0;1742}1743 1744static const struct coresight_ops_sink tmc_etr_sink_ops = {1745 .enable = tmc_enable_etr_sink,1746 .disable = tmc_disable_etr_sink,1747 .alloc_buffer = tmc_alloc_etr_buffer,1748 .update_buffer = tmc_update_etr_buffer,1749 .free_buffer = tmc_free_etr_buffer,1750};1751 1752const struct coresight_ops tmc_etr_cs_ops = {1753 .sink_ops = &tmc_etr_sink_ops,1754};1755 1756int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)1757{1758 int ret = 0;1759 unsigned long flags;1760 1761 /* config types are set a boot time and never change */1762 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))1763 return -EINVAL;1764 1765 spin_lock_irqsave(&drvdata->spinlock, flags);1766 if (drvdata->reading) {1767 ret = -EBUSY;1768 goto out;1769 }1770 1771 /*1772 * We can safely allow reads even if the ETR is operating in PERF mode,1773 * since the sysfs session is captured in mode specific data.1774 * If drvdata::sysfs_data is NULL the trace data has been read already.1775 */1776 if (!drvdata->sysfs_buf) {1777 ret = -EINVAL;1778 goto out;1779 }1780 1781 /* Disable the TMC if we are trying to read from a running session. */1782 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS)1783 __tmc_etr_disable_hw(drvdata);1784 1785 drvdata->reading = true;1786out:1787 spin_unlock_irqrestore(&drvdata->spinlock, flags);1788 1789 return ret;1790}1791 1792int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)1793{1794 unsigned long flags;1795 struct etr_buf *sysfs_buf = NULL;1796 1797 /* config types are set a boot time and never change */1798 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))1799 return -EINVAL;1800 1801 spin_lock_irqsave(&drvdata->spinlock, flags);1802 1803 /* RE-enable the TMC if need be */1804 if (coresight_get_mode(drvdata->csdev) == CS_MODE_SYSFS) {1805 /*1806 * The trace run will continue with the same allocated trace1807 * buffer. Since the tracer is still enabled drvdata::buf can't1808 * be NULL.1809 */1810 __tmc_etr_enable_hw(drvdata);1811 } else {1812 /*1813 * The ETR is not tracing and the buffer was just read.1814 * As such prepare to free the trace buffer.1815 */1816 sysfs_buf = drvdata->sysfs_buf;1817 drvdata->sysfs_buf = NULL;1818 }1819 1820 drvdata->reading = false;1821 spin_unlock_irqrestore(&drvdata->spinlock, flags);1822 1823 /* Free allocated memory out side of the spinlock */1824 if (sysfs_buf)1825 tmc_etr_free_sysfs_buf(sysfs_buf);1826 1827 return 0;1828}1829 1830static const char *const buf_modes_str[] = {1831 [ETR_MODE_FLAT] = "flat",1832 [ETR_MODE_ETR_SG] = "tmc-sg",1833 [ETR_MODE_CATU] = "catu",1834 [ETR_MODE_AUTO] = "auto",1835};1836 1837static ssize_t buf_modes_available_show(struct device *dev,1838 struct device_attribute *attr, char *buf)1839{1840 struct etr_buf_hw buf_hw;1841 ssize_t size = 0;1842 1843 get_etr_buf_hw(dev, &buf_hw);1844 size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);1845 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);1846 if (buf_hw.has_etr_sg)1847 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);1848 1849 if (buf_hw.has_catu)1850 size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);1851 1852 size += sysfs_emit_at(buf, size, "\n");1853 return size;1854}1855static DEVICE_ATTR_RO(buf_modes_available);1856 1857static ssize_t buf_mode_preferred_show(struct device *dev,1858 struct device_attribute *attr, char *buf)1859{1860 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);1861 1862 return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);1863}1864 1865static ssize_t buf_mode_preferred_store(struct device *dev,1866 struct device_attribute *attr,1867 const char *buf, size_t size)1868{1869 struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);1870 struct etr_buf_hw buf_hw;1871 1872 get_etr_buf_hw(dev, &buf_hw);1873 if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))1874 drvdata->etr_mode = ETR_MODE_FLAT;1875 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)1876 drvdata->etr_mode = ETR_MODE_ETR_SG;1877 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)1878 drvdata->etr_mode = ETR_MODE_CATU;1879 else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))1880 drvdata->etr_mode = ETR_MODE_AUTO;1881 else1882 return -EINVAL;1883 return size;1884}1885static DEVICE_ATTR_RW(buf_mode_preferred);1886 1887static struct attribute *coresight_etr_attrs[] = {1888 &dev_attr_buf_modes_available.attr,1889 &dev_attr_buf_mode_preferred.attr,1890 NULL,1891};1892 1893const struct attribute_group coresight_etr_group = {1894 .attrs = coresight_etr_attrs,1895};1896