2214 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Intel(R) Trace Hub Memory Storage Unit4 *5 * Copyright (C) 2014-2015 Intel Corporation.6 */7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt9 10#include <linux/types.h>11#include <linux/module.h>12#include <linux/device.h>13#include <linux/uaccess.h>14#include <linux/sizes.h>15#include <linux/printk.h>16#include <linux/slab.h>17#include <linux/mm.h>18#include <linux/fs.h>19#include <linux/io.h>20#include <linux/workqueue.h>21#include <linux/dma-mapping.h>22 23#ifdef CONFIG_X8624#include <asm/set_memory.h>25#endif26 27#include <linux/intel_th.h>28#include "intel_th.h"29#include "msu.h"30 31#define msc_dev(x) (&(x)->thdev->dev)32 33/*34 * Lockout state transitions:35 * READY -> INUSE -+-> LOCKED -+-> READY -> etc.36 * \-----------/37 * WIN_READY: window can be used by HW38 * WIN_INUSE: window is in use39 * WIN_LOCKED: window is filled up and is being processed by the buffer40 * handling code41 *42 * All state transitions happen automatically, except for the LOCKED->READY,43 * which needs to be signalled by the buffer code by calling44 * intel_th_msc_window_unlock().45 *46 * When the interrupt handler has to switch to the next window, it checks47 * whether it's READY, and if it is, it performs the switch and tracing48 * continues. If it's LOCKED, it stops the trace.49 */50enum lockout_state {51 WIN_READY = 0,52 WIN_INUSE,53 WIN_LOCKED54};55 56/**57 * struct msc_window - multiblock mode window descriptor58 * @entry: window list linkage (msc::win_list)59 * @pgoff: page offset into the buffer that this window starts at60 * @lockout: lockout state, see comment below61 * @lo_lock: lockout state serialization62 * @nr_blocks: number of blocks (pages) in this window63 * @nr_segs: number of segments in this window (<= @nr_blocks)64 * @msc: pointer to the MSC device65 * @_sgt: array of block descriptors66 * @sgt: array of block descriptors67 */68struct msc_window {69 struct list_head entry;70 unsigned long pgoff;71 enum lockout_state lockout;72 spinlock_t lo_lock;73 unsigned int nr_blocks;74 unsigned int nr_segs;75 struct msc *msc;76 struct sg_table _sgt;77 struct sg_table *sgt;78};79 80/**81 * struct msc_iter - iterator for msc buffer82 * @entry: msc::iter_list linkage83 * @msc: pointer to the MSC device84 * @start_win: oldest window85 * @win: current window86 * @offset: current logical offset into the buffer87 * @start_block: oldest block in the window88 * @block: block number in the window89 * @block_off: offset into current block90 * @wrap_count: block wrapping handling91 * @eof: end of buffer reached92 */93struct msc_iter {94 struct list_head entry;95 struct msc *msc;96 struct msc_window *start_win;97 struct msc_window *win;98 unsigned long offset;99 struct scatterlist *start_block;100 struct scatterlist *block;101 unsigned int block_off;102 unsigned int wrap_count;103 unsigned int eof;104};105 106/**107 * struct msc - MSC device representation108 * @reg_base: register window base address109 * @thdev: intel_th_device pointer110 * @mbuf: MSU buffer, if assigned111 * @mbuf_priv MSU buffer's private data, if @mbuf112 * @win_list: list of windows in multiblock mode113 * @single_sgt: single mode buffer114 * @cur_win: current window115 * @nr_pages: total number of pages allocated for this buffer116 * @single_sz: amount of data in single mode117 * @single_wrap: single mode wrap occurred118 * @base: buffer's base pointer119 * @base_addr: buffer's base address120 * @user_count: number of users of the buffer121 * @mmap_count: number of mappings122 * @buf_mutex: mutex to serialize access to buffer-related bits123 * @enabled: MSC is enabled124 * @wrap: wrapping is enabled125 * @mode: MSC operating mode126 * @burst_len: write burst length127 * @index: number of this MSC in the MSU128 */129struct msc {130 void __iomem *reg_base;131 void __iomem *msu_base;132 struct intel_th_device *thdev;133 134 const struct msu_buffer *mbuf;135 void *mbuf_priv;136 137 struct work_struct work;138 struct list_head win_list;139 struct sg_table single_sgt;140 struct msc_window *cur_win;141 struct msc_window *switch_on_unlock;142 unsigned long nr_pages;143 unsigned long single_sz;144 unsigned int single_wrap : 1;145 void *base;146 dma_addr_t base_addr;147 u32 orig_addr;148 u32 orig_sz;149 150 /* <0: no buffer, 0: no users, >0: active users */151 atomic_t user_count;152 153 atomic_t mmap_count;154 struct mutex buf_mutex;155 156 struct list_head iter_list;157 158 bool stop_on_full;159 160 /* config */161 unsigned int enabled : 1,162 wrap : 1,163 do_irq : 1,164 multi_is_broken : 1;165 unsigned int mode;166 unsigned int burst_len;167 unsigned int index;168};169 170static LIST_HEAD(msu_buffer_list);171static DEFINE_MUTEX(msu_buffer_mutex);172 173/**174 * struct msu_buffer_entry - internal MSU buffer bookkeeping175 * @entry: link to msu_buffer_list176 * @mbuf: MSU buffer object177 * @owner: module that provides this MSU buffer178 */179struct msu_buffer_entry {180 struct list_head entry;181 const struct msu_buffer *mbuf;182 struct module *owner;183};184 185static struct msu_buffer_entry *__msu_buffer_entry_find(const char *name)186{187 struct msu_buffer_entry *mbe;188 189 lockdep_assert_held(&msu_buffer_mutex);190 191 list_for_each_entry(mbe, &msu_buffer_list, entry) {192 if (!strcmp(mbe->mbuf->name, name))193 return mbe;194 }195 196 return NULL;197}198 199static const struct msu_buffer *200msu_buffer_get(const char *name)201{202 struct msu_buffer_entry *mbe;203 204 mutex_lock(&msu_buffer_mutex);205 mbe = __msu_buffer_entry_find(name);206 if (mbe && !try_module_get(mbe->owner))207 mbe = NULL;208 mutex_unlock(&msu_buffer_mutex);209 210 return mbe ? mbe->mbuf : NULL;211}212 213static void msu_buffer_put(const struct msu_buffer *mbuf)214{215 struct msu_buffer_entry *mbe;216 217 mutex_lock(&msu_buffer_mutex);218 mbe = __msu_buffer_entry_find(mbuf->name);219 if (mbe)220 module_put(mbe->owner);221 mutex_unlock(&msu_buffer_mutex);222}223 224int intel_th_msu_buffer_register(const struct msu_buffer *mbuf,225 struct module *owner)226{227 struct msu_buffer_entry *mbe;228 int ret = 0;229 230 mbe = kzalloc(sizeof(*mbe), GFP_KERNEL);231 if (!mbe)232 return -ENOMEM;233 234 mutex_lock(&msu_buffer_mutex);235 if (__msu_buffer_entry_find(mbuf->name)) {236 ret = -EEXIST;237 kfree(mbe);238 goto unlock;239 }240 241 mbe->mbuf = mbuf;242 mbe->owner = owner;243 list_add_tail(&mbe->entry, &msu_buffer_list);244unlock:245 mutex_unlock(&msu_buffer_mutex);246 247 return ret;248}249EXPORT_SYMBOL_GPL(intel_th_msu_buffer_register);250 251void intel_th_msu_buffer_unregister(const struct msu_buffer *mbuf)252{253 struct msu_buffer_entry *mbe;254 255 mutex_lock(&msu_buffer_mutex);256 mbe = __msu_buffer_entry_find(mbuf->name);257 if (mbe) {258 list_del(&mbe->entry);259 kfree(mbe);260 }261 mutex_unlock(&msu_buffer_mutex);262}263EXPORT_SYMBOL_GPL(intel_th_msu_buffer_unregister);264 265static inline bool msc_block_is_empty(struct msc_block_desc *bdesc)266{267 /* header hasn't been written */268 if (!bdesc->valid_dw)269 return true;270 271 /* valid_dw includes the header */272 if (!msc_data_sz(bdesc))273 return true;274 275 return false;276}277 278static inline struct scatterlist *msc_win_base_sg(struct msc_window *win)279{280 return win->sgt->sgl;281}282 283static inline struct msc_block_desc *msc_win_base(struct msc_window *win)284{285 return sg_virt(msc_win_base_sg(win));286}287 288static inline dma_addr_t msc_win_base_dma(struct msc_window *win)289{290 return sg_dma_address(msc_win_base_sg(win));291}292 293static inline unsigned long294msc_win_base_pfn(struct msc_window *win)295{296 return PFN_DOWN(msc_win_base_dma(win));297}298 299/**300 * msc_is_last_win() - check if a window is the last one for a given MSC301 * @win: window302 * Return: true if @win is the last window in MSC's multiblock buffer303 */304static inline bool msc_is_last_win(struct msc_window *win)305{306 return win->entry.next == &win->msc->win_list;307}308 309/**310 * msc_next_window() - return next window in the multiblock buffer311 * @win: current window312 *313 * Return: window following the current one314 */315static struct msc_window *msc_next_window(struct msc_window *win)316{317 if (msc_is_last_win(win))318 return list_first_entry(&win->msc->win_list, struct msc_window,319 entry);320 321 return list_next_entry(win, entry);322}323 324static size_t msc_win_total_sz(struct msc_window *win)325{326 struct scatterlist *sg;327 unsigned int blk;328 size_t size = 0;329 330 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {331 struct msc_block_desc *bdesc = sg_virt(sg);332 333 if (msc_block_wrapped(bdesc))334 return (size_t)win->nr_blocks << PAGE_SHIFT;335 336 size += msc_total_sz(bdesc);337 if (msc_block_last_written(bdesc))338 break;339 }340 341 return size;342}343 344/**345 * msc_find_window() - find a window matching a given sg_table346 * @msc: MSC device347 * @sgt: SG table of the window348 * @nonempty: skip over empty windows349 *350 * Return: MSC window structure pointer or NULL if the window351 * could not be found.352 */353static struct msc_window *354msc_find_window(struct msc *msc, struct sg_table *sgt, bool nonempty)355{356 struct msc_window *win;357 unsigned int found = 0;358 359 if (list_empty(&msc->win_list))360 return NULL;361 362 /*363 * we might need a radix tree for this, depending on how364 * many windows a typical user would allocate; ideally it's365 * something like 2, in which case we're good366 */367 list_for_each_entry(win, &msc->win_list, entry) {368 if (win->sgt == sgt)369 found++;370 371 /* skip the empty ones */372 if (nonempty && msc_block_is_empty(msc_win_base(win)))373 continue;374 375 if (found)376 return win;377 }378 379 return NULL;380}381 382/**383 * msc_oldest_window() - locate the window with oldest data384 * @msc: MSC device385 *386 * This should only be used in multiblock mode. Caller should hold the387 * msc::user_count reference.388 *389 * Return: the oldest window with valid data390 */391static struct msc_window *msc_oldest_window(struct msc *msc)392{393 struct msc_window *win;394 395 if (list_empty(&msc->win_list))396 return NULL;397 398 win = msc_find_window(msc, msc_next_window(msc->cur_win)->sgt, true);399 if (win)400 return win;401 402 return list_first_entry(&msc->win_list, struct msc_window, entry);403}404 405/**406 * msc_win_oldest_sg() - locate the oldest block in a given window407 * @win: window to look at408 *409 * Return: index of the block with the oldest data410 */411static struct scatterlist *msc_win_oldest_sg(struct msc_window *win)412{413 unsigned int blk;414 struct scatterlist *sg;415 struct msc_block_desc *bdesc = msc_win_base(win);416 417 /* without wrapping, first block is the oldest */418 if (!msc_block_wrapped(bdesc))419 return msc_win_base_sg(win);420 421 /*422 * with wrapping, last written block contains both the newest and the423 * oldest data for this window.424 */425 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {426 struct msc_block_desc *bdesc = sg_virt(sg);427 428 if (msc_block_last_written(bdesc))429 return sg;430 }431 432 return msc_win_base_sg(win);433}434 435static struct msc_block_desc *msc_iter_bdesc(struct msc_iter *iter)436{437 return sg_virt(iter->block);438}439 440static struct msc_iter *msc_iter_install(struct msc *msc)441{442 struct msc_iter *iter;443 444 iter = kzalloc(sizeof(*iter), GFP_KERNEL);445 if (!iter)446 return ERR_PTR(-ENOMEM);447 448 mutex_lock(&msc->buf_mutex);449 450 /*451 * Reading and tracing are mutually exclusive; if msc is452 * enabled, open() will fail; otherwise existing readers453 * will prevent enabling the msc and the rest of fops don't454 * need to worry about it.455 */456 if (msc->enabled) {457 kfree(iter);458 iter = ERR_PTR(-EBUSY);459 goto unlock;460 }461 462 iter->msc = msc;463 464 list_add_tail(&iter->entry, &msc->iter_list);465unlock:466 mutex_unlock(&msc->buf_mutex);467 468 return iter;469}470 471static void msc_iter_remove(struct msc_iter *iter, struct msc *msc)472{473 mutex_lock(&msc->buf_mutex);474 list_del(&iter->entry);475 mutex_unlock(&msc->buf_mutex);476 477 kfree(iter);478}479 480static void msc_iter_block_start(struct msc_iter *iter)481{482 if (iter->start_block)483 return;484 485 iter->start_block = msc_win_oldest_sg(iter->win);486 iter->block = iter->start_block;487 iter->wrap_count = 0;488 489 /*490 * start with the block with oldest data; if data has wrapped491 * in this window, it should be in this block492 */493 if (msc_block_wrapped(msc_iter_bdesc(iter)))494 iter->wrap_count = 2;495 496}497 498static int msc_iter_win_start(struct msc_iter *iter, struct msc *msc)499{500 /* already started, nothing to do */501 if (iter->start_win)502 return 0;503 504 iter->start_win = msc_oldest_window(msc);505 if (!iter->start_win)506 return -EINVAL;507 508 iter->win = iter->start_win;509 iter->start_block = NULL;510 511 msc_iter_block_start(iter);512 513 return 0;514}515 516static int msc_iter_win_advance(struct msc_iter *iter)517{518 iter->win = msc_next_window(iter->win);519 iter->start_block = NULL;520 521 if (iter->win == iter->start_win) {522 iter->eof++;523 return 1;524 }525 526 msc_iter_block_start(iter);527 528 return 0;529}530 531static int msc_iter_block_advance(struct msc_iter *iter)532{533 iter->block_off = 0;534 535 /* wrapping */536 if (iter->wrap_count && iter->block == iter->start_block) {537 iter->wrap_count--;538 if (!iter->wrap_count)539 /* copied newest data from the wrapped block */540 return msc_iter_win_advance(iter);541 }542 543 /* no wrapping, check for last written block */544 if (!iter->wrap_count && msc_block_last_written(msc_iter_bdesc(iter)))545 /* copied newest data for the window */546 return msc_iter_win_advance(iter);547 548 /* block advance */549 if (sg_is_last(iter->block))550 iter->block = msc_win_base_sg(iter->win);551 else552 iter->block = sg_next(iter->block);553 554 /* no wrapping, sanity check in case there is no last written block */555 if (!iter->wrap_count && iter->block == iter->start_block)556 return msc_iter_win_advance(iter);557 558 return 0;559}560 561/**562 * msc_buffer_iterate() - go through multiblock buffer's data563 * @iter: iterator structure564 * @size: amount of data to scan565 * @data: callback's private data566 * @fn: iterator callback567 *568 * This will start at the window which will be written to next (containing569 * the oldest data) and work its way to the current window, calling @fn570 * for each chunk of data as it goes.571 *572 * Caller should have msc::user_count reference to make sure the buffer573 * doesn't disappear from under us.574 *575 * Return: amount of data actually scanned.576 */577static ssize_t578msc_buffer_iterate(struct msc_iter *iter, size_t size, void *data,579 unsigned long (*fn)(void *, void *, size_t))580{581 struct msc *msc = iter->msc;582 size_t len = size;583 unsigned int advance;584 585 if (iter->eof)586 return 0;587 588 /* start with the oldest window */589 if (msc_iter_win_start(iter, msc))590 return 0;591 592 do {593 unsigned long data_bytes = msc_data_sz(msc_iter_bdesc(iter));594 void *src = (void *)msc_iter_bdesc(iter) + MSC_BDESC;595 size_t tocopy = data_bytes, copied = 0;596 size_t remaining = 0;597 598 advance = 1;599 600 /*601 * If block wrapping happened, we need to visit the last block602 * twice, because it contains both the oldest and the newest603 * data in this window.604 *605 * First time (wrap_count==2), in the very beginning, to collect606 * the oldest data, which is in the range607 * (data_bytes..DATA_IN_PAGE).608 *609 * Second time (wrap_count==1), it's just like any other block,610 * containing data in the range of [MSC_BDESC..data_bytes].611 */612 if (iter->block == iter->start_block && iter->wrap_count == 2) {613 tocopy = DATA_IN_PAGE - data_bytes;614 src += data_bytes;615 }616 617 if (!tocopy)618 goto next_block;619 620 tocopy -= iter->block_off;621 src += iter->block_off;622 623 if (len < tocopy) {624 tocopy = len;625 advance = 0;626 }627 628 remaining = fn(data, src, tocopy);629 630 if (remaining)631 advance = 0;632 633 copied = tocopy - remaining;634 len -= copied;635 iter->block_off += copied;636 iter->offset += copied;637 638 if (!advance)639 break;640 641next_block:642 if (msc_iter_block_advance(iter))643 break;644 645 } while (len);646 647 return size - len;648}649 650/**651 * msc_buffer_clear_hw_header() - clear hw header for multiblock652 * @msc: MSC device653 */654static void msc_buffer_clear_hw_header(struct msc *msc)655{656 struct msc_window *win;657 struct scatterlist *sg;658 659 list_for_each_entry(win, &msc->win_list, entry) {660 unsigned int blk;661 662 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {663 struct msc_block_desc *bdesc = sg_virt(sg);664 665 memset_startat(bdesc, 0, hw_tag);666 }667 }668}669 670static int intel_th_msu_init(struct msc *msc)671{672 u32 mintctl, msusts;673 674 if (!msc->do_irq)675 return 0;676 677 if (!msc->mbuf)678 return 0;679 680 mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);681 mintctl |= msc->index ? M1BLIE : M0BLIE;682 iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);683 if (mintctl != ioread32(msc->msu_base + REG_MSU_MINTCTL)) {684 dev_info(msc_dev(msc), "MINTCTL ignores writes: no usable interrupts\n");685 msc->do_irq = 0;686 return 0;687 }688 689 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);690 iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);691 692 return 0;693}694 695static void intel_th_msu_deinit(struct msc *msc)696{697 u32 mintctl;698 699 if (!msc->do_irq)700 return;701 702 mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);703 mintctl &= msc->index ? ~M1BLIE : ~M0BLIE;704 iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);705}706 707static int msc_win_set_lockout(struct msc_window *win,708 enum lockout_state expect,709 enum lockout_state new)710{711 enum lockout_state old;712 unsigned long flags;713 int ret = 0;714 715 if (!win->msc->mbuf)716 return 0;717 718 spin_lock_irqsave(&win->lo_lock, flags);719 old = win->lockout;720 721 if (old != expect) {722 ret = -EINVAL;723 goto unlock;724 }725 726 win->lockout = new;727 728 if (old == expect && new == WIN_LOCKED)729 atomic_inc(&win->msc->user_count);730 else if (old == expect && old == WIN_LOCKED)731 atomic_dec(&win->msc->user_count);732 733unlock:734 spin_unlock_irqrestore(&win->lo_lock, flags);735 736 if (ret) {737 if (expect == WIN_READY && old == WIN_LOCKED)738 return -EBUSY;739 740 /* from intel_th_msc_window_unlock(), don't warn if not locked */741 if (expect == WIN_LOCKED && old == new)742 return 0;743 744 dev_warn_ratelimited(msc_dev(win->msc),745 "expected lockout state %d, got %d\n",746 expect, old);747 }748 749 return ret;750}751/**752 * msc_configure() - set up MSC hardware753 * @msc: the MSC device to configure754 *755 * Program storage mode, wrapping, burst length and trace buffer address756 * into a given MSC. Then, enable tracing and set msc::enabled.757 * The latter is serialized on msc::buf_mutex, so make sure to hold it.758 *759 * Return: %0 for success or a negative error code otherwise.760 */761static int msc_configure(struct msc *msc)762{763 u32 reg;764 765 lockdep_assert_held(&msc->buf_mutex);766 767 if (msc->mode > MSC_MODE_MULTI)768 return -EINVAL;769 770 if (msc->mode == MSC_MODE_MULTI) {771 if (msc_win_set_lockout(msc->cur_win, WIN_READY, WIN_INUSE))772 return -EBUSY;773 774 msc_buffer_clear_hw_header(msc);775 }776 777 msc->orig_addr = ioread32(msc->reg_base + REG_MSU_MSC0BAR);778 msc->orig_sz = ioread32(msc->reg_base + REG_MSU_MSC0SIZE);779 780 reg = msc->base_addr >> PAGE_SHIFT;781 iowrite32(reg, msc->reg_base + REG_MSU_MSC0BAR);782 783 if (msc->mode == MSC_MODE_SINGLE) {784 reg = msc->nr_pages;785 iowrite32(reg, msc->reg_base + REG_MSU_MSC0SIZE);786 }787 788 reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);789 reg &= ~(MSC_MODE | MSC_WRAPEN | MSC_EN | MSC_RD_HDR_OVRD);790 791 reg |= MSC_EN;792 reg |= msc->mode << __ffs(MSC_MODE);793 reg |= msc->burst_len << __ffs(MSC_LEN);794 795 if (msc->wrap)796 reg |= MSC_WRAPEN;797 798 iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);799 800 intel_th_msu_init(msc);801 802 msc->thdev->output.multiblock = msc->mode == MSC_MODE_MULTI;803 intel_th_trace_enable(msc->thdev);804 msc->enabled = 1;805 806 if (msc->mbuf && msc->mbuf->activate)807 msc->mbuf->activate(msc->mbuf_priv);808 809 return 0;810}811 812/**813 * msc_disable() - disable MSC hardware814 * @msc: MSC device to disable815 *816 * If @msc is enabled, disable tracing on the switch and then disable MSC817 * storage. Caller must hold msc::buf_mutex.818 */819static void msc_disable(struct msc *msc)820{821 struct msc_window *win = msc->cur_win;822 u32 reg;823 824 lockdep_assert_held(&msc->buf_mutex);825 826 if (msc->mode == MSC_MODE_MULTI)827 msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);828 829 if (msc->mbuf && msc->mbuf->deactivate)830 msc->mbuf->deactivate(msc->mbuf_priv);831 intel_th_msu_deinit(msc);832 intel_th_trace_disable(msc->thdev);833 834 if (msc->mode == MSC_MODE_SINGLE) {835 reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);836 msc->single_wrap = !!(reg & MSCSTS_WRAPSTAT);837 838 reg = ioread32(msc->reg_base + REG_MSU_MSC0MWP);839 msc->single_sz = reg & ((msc->nr_pages << PAGE_SHIFT) - 1);840 dev_dbg(msc_dev(msc), "MSCnMWP: %08x/%08lx, wrap: %d\n",841 reg, msc->single_sz, msc->single_wrap);842 }843 844 reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);845 reg &= ~MSC_EN;846 iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);847 848 if (msc->mbuf && msc->mbuf->ready)849 msc->mbuf->ready(msc->mbuf_priv, win->sgt,850 msc_win_total_sz(win));851 852 msc->enabled = 0;853 854 iowrite32(msc->orig_addr, msc->reg_base + REG_MSU_MSC0BAR);855 iowrite32(msc->orig_sz, msc->reg_base + REG_MSU_MSC0SIZE);856 857 dev_dbg(msc_dev(msc), "MSCnNWSA: %08x\n",858 ioread32(msc->reg_base + REG_MSU_MSC0NWSA));859 860 reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);861 dev_dbg(msc_dev(msc), "MSCnSTS: %08x\n", reg);862 863 reg = ioread32(msc->reg_base + REG_MSU_MSUSTS);864 reg &= msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;865 iowrite32(reg, msc->reg_base + REG_MSU_MSUSTS);866}867 868static int intel_th_msc_activate(struct intel_th_device *thdev)869{870 struct msc *msc = dev_get_drvdata(&thdev->dev);871 int ret = -EBUSY;872 873 if (!atomic_inc_unless_negative(&msc->user_count))874 return -ENODEV;875 876 mutex_lock(&msc->buf_mutex);877 878 /* if there are readers, refuse */879 if (list_empty(&msc->iter_list))880 ret = msc_configure(msc);881 882 mutex_unlock(&msc->buf_mutex);883 884 if (ret)885 atomic_dec(&msc->user_count);886 887 return ret;888}889 890static void intel_th_msc_deactivate(struct intel_th_device *thdev)891{892 struct msc *msc = dev_get_drvdata(&thdev->dev);893 894 mutex_lock(&msc->buf_mutex);895 if (msc->enabled) {896 msc_disable(msc);897 atomic_dec(&msc->user_count);898 }899 mutex_unlock(&msc->buf_mutex);900}901 902/**903 * msc_buffer_contig_alloc() - allocate a contiguous buffer for SINGLE mode904 * @msc: MSC device905 * @size: allocation size in bytes906 *907 * This modifies msc::base, which requires msc::buf_mutex to serialize, so the908 * caller is expected to hold it.909 *910 * Return: 0 on success, -errno otherwise.911 */912static int msc_buffer_contig_alloc(struct msc *msc, unsigned long size)913{914 unsigned long nr_pages = size >> PAGE_SHIFT;915 unsigned int order = get_order(size);916 struct page *page;917 int ret;918 919 if (!size)920 return 0;921 922 ret = sg_alloc_table(&msc->single_sgt, 1, GFP_KERNEL);923 if (ret)924 goto err_out;925 926 ret = -ENOMEM;927 page = alloc_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, order);928 if (!page)929 goto err_free_sgt;930 931 split_page(page, order);932 sg_set_buf(msc->single_sgt.sgl, page_address(page), size);933 934 ret = dma_map_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, 1,935 DMA_FROM_DEVICE);936 if (ret < 0)937 goto err_free_pages;938 939 msc->nr_pages = nr_pages;940 msc->base = page_address(page);941 msc->base_addr = sg_dma_address(msc->single_sgt.sgl);942 943 return 0;944 945err_free_pages:946 __free_pages(page, order);947 948err_free_sgt:949 sg_free_table(&msc->single_sgt);950 951err_out:952 return ret;953}954 955/**956 * msc_buffer_contig_free() - free a contiguous buffer957 * @msc: MSC configured in SINGLE mode958 */959static void msc_buffer_contig_free(struct msc *msc)960{961 unsigned long off;962 963 dma_unmap_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl,964 1, DMA_FROM_DEVICE);965 sg_free_table(&msc->single_sgt);966 967 for (off = 0; off < msc->nr_pages << PAGE_SHIFT; off += PAGE_SIZE) {968 struct page *page = virt_to_page(msc->base + off);969 970 page->mapping = NULL;971 __free_page(page);972 }973 974 msc->nr_pages = 0;975}976 977/**978 * msc_buffer_contig_get_page() - find a page at a given offset979 * @msc: MSC configured in SINGLE mode980 * @pgoff: page offset981 *982 * Return: page, if @pgoff is within the range, NULL otherwise.983 */984static struct page *msc_buffer_contig_get_page(struct msc *msc,985 unsigned long pgoff)986{987 if (pgoff >= msc->nr_pages)988 return NULL;989 990 return virt_to_page(msc->base + (pgoff << PAGE_SHIFT));991}992 993static int __msc_buffer_win_alloc(struct msc_window *win,994 unsigned int nr_segs)995{996 struct scatterlist *sg_ptr;997 void *block;998 int i, ret;999 1000 ret = sg_alloc_table(win->sgt, nr_segs, GFP_KERNEL);1001 if (ret)1002 return -ENOMEM;1003 1004 for_each_sg(win->sgt->sgl, sg_ptr, nr_segs, i) {1005 block = dma_alloc_coherent(msc_dev(win->msc)->parent->parent,1006 PAGE_SIZE, &sg_dma_address(sg_ptr),1007 GFP_KERNEL);1008 if (!block)1009 goto err_nomem;1010 1011 sg_set_buf(sg_ptr, block, PAGE_SIZE);1012 }1013 1014 return nr_segs;1015 1016err_nomem:1017 for_each_sg(win->sgt->sgl, sg_ptr, i, ret)1018 dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,1019 sg_virt(sg_ptr), sg_dma_address(sg_ptr));1020 1021 sg_free_table(win->sgt);1022 1023 return -ENOMEM;1024}1025 1026#ifdef CONFIG_X861027static void msc_buffer_set_uc(struct msc *msc)1028{1029 struct scatterlist *sg_ptr;1030 struct msc_window *win;1031 int i;1032 1033 if (msc->mode == MSC_MODE_SINGLE) {1034 set_memory_uc((unsigned long)msc->base, msc->nr_pages);1035 return;1036 }1037 1038 list_for_each_entry(win, &msc->win_list, entry) {1039 for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) {1040 /* Set the page as uncached */1041 set_memory_uc((unsigned long)sg_virt(sg_ptr),1042 PFN_DOWN(sg_ptr->length));1043 }1044 }1045}1046 1047static void msc_buffer_set_wb(struct msc *msc)1048{1049 struct scatterlist *sg_ptr;1050 struct msc_window *win;1051 int i;1052 1053 if (msc->mode == MSC_MODE_SINGLE) {1054 set_memory_wb((unsigned long)msc->base, msc->nr_pages);1055 return;1056 }1057 1058 list_for_each_entry(win, &msc->win_list, entry) {1059 for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) {1060 /* Reset the page to write-back */1061 set_memory_wb((unsigned long)sg_virt(sg_ptr),1062 PFN_DOWN(sg_ptr->length));1063 }1064 }1065}1066#else /* !X86 */1067static inline void1068msc_buffer_set_uc(struct msc *msc) {}1069static inline void msc_buffer_set_wb(struct msc *msc) {}1070#endif /* CONFIG_X86 */1071 1072static struct page *msc_sg_page(struct scatterlist *sg)1073{1074 void *addr = sg_virt(sg);1075 1076 if (is_vmalloc_addr(addr))1077 return vmalloc_to_page(addr);1078 1079 return sg_page(sg);1080}1081 1082/**1083 * msc_buffer_win_alloc() - alloc a window for a multiblock mode1084 * @msc: MSC device1085 * @nr_blocks: number of pages in this window1086 *1087 * This modifies msc::win_list and msc::base, which requires msc::buf_mutex1088 * to serialize, so the caller is expected to hold it.1089 *1090 * Return: 0 on success, -errno otherwise.1091 */1092static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)1093{1094 struct msc_window *win;1095 int ret = -ENOMEM;1096 1097 if (!nr_blocks)1098 return 0;1099 1100 win = kzalloc(sizeof(*win), GFP_KERNEL);1101 if (!win)1102 return -ENOMEM;1103 1104 win->msc = msc;1105 win->sgt = &win->_sgt;1106 win->lockout = WIN_READY;1107 spin_lock_init(&win->lo_lock);1108 1109 if (!list_empty(&msc->win_list)) {1110 struct msc_window *prev = list_last_entry(&msc->win_list,1111 struct msc_window,1112 entry);1113 1114 win->pgoff = prev->pgoff + prev->nr_blocks;1115 }1116 1117 if (msc->mbuf && msc->mbuf->alloc_window)1118 ret = msc->mbuf->alloc_window(msc->mbuf_priv, &win->sgt,1119 nr_blocks << PAGE_SHIFT);1120 else1121 ret = __msc_buffer_win_alloc(win, nr_blocks);1122 1123 if (ret <= 0)1124 goto err_nomem;1125 1126 win->nr_segs = ret;1127 win->nr_blocks = nr_blocks;1128 1129 if (list_empty(&msc->win_list)) {1130 msc->base = msc_win_base(win);1131 msc->base_addr = msc_win_base_dma(win);1132 msc->cur_win = win;1133 }1134 1135 list_add_tail(&win->entry, &msc->win_list);1136 msc->nr_pages += nr_blocks;1137 1138 return 0;1139 1140err_nomem:1141 kfree(win);1142 1143 return ret;1144}1145 1146static void __msc_buffer_win_free(struct msc *msc, struct msc_window *win)1147{1148 struct scatterlist *sg;1149 int i;1150 1151 for_each_sg(win->sgt->sgl, sg, win->nr_segs, i) {1152 struct page *page = msc_sg_page(sg);1153 1154 page->mapping = NULL;1155 dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,1156 sg_virt(sg), sg_dma_address(sg));1157 }1158 sg_free_table(win->sgt);1159}1160 1161/**1162 * msc_buffer_win_free() - free a window from MSC's window list1163 * @msc: MSC device1164 * @win: window to free1165 *1166 * This modifies msc::win_list and msc::base, which requires msc::buf_mutex1167 * to serialize, so the caller is expected to hold it.1168 */1169static void msc_buffer_win_free(struct msc *msc, struct msc_window *win)1170{1171 msc->nr_pages -= win->nr_blocks;1172 1173 list_del(&win->entry);1174 if (list_empty(&msc->win_list)) {1175 msc->base = NULL;1176 msc->base_addr = 0;1177 }1178 1179 if (msc->mbuf && msc->mbuf->free_window)1180 msc->mbuf->free_window(msc->mbuf_priv, win->sgt);1181 else1182 __msc_buffer_win_free(msc, win);1183 1184 kfree(win);1185}1186 1187/**1188 * msc_buffer_relink() - set up block descriptors for multiblock mode1189 * @msc: MSC device1190 *1191 * This traverses msc::win_list, which requires msc::buf_mutex to serialize,1192 * so the caller is expected to hold it.1193 */1194static void msc_buffer_relink(struct msc *msc)1195{1196 struct msc_window *win, *next_win;1197 1198 /* call with msc::mutex locked */1199 list_for_each_entry(win, &msc->win_list, entry) {1200 struct scatterlist *sg;1201 unsigned int blk;1202 u32 sw_tag = 0;1203 1204 /*1205 * Last window's next_win should point to the first window1206 * and MSC_SW_TAG_LASTWIN should be set.1207 */1208 if (msc_is_last_win(win)) {1209 sw_tag |= MSC_SW_TAG_LASTWIN;1210 next_win = list_first_entry(&msc->win_list,1211 struct msc_window, entry);1212 } else {1213 next_win = list_next_entry(win, entry);1214 }1215 1216 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {1217 struct msc_block_desc *bdesc = sg_virt(sg);1218 1219 memset(bdesc, 0, sizeof(*bdesc));1220 1221 bdesc->next_win = msc_win_base_pfn(next_win);1222 1223 /*1224 * Similarly to last window, last block should point1225 * to the first one.1226 */1227 if (blk == win->nr_segs - 1) {1228 sw_tag |= MSC_SW_TAG_LASTBLK;1229 bdesc->next_blk = msc_win_base_pfn(win);1230 } else {1231 dma_addr_t addr = sg_dma_address(sg_next(sg));1232 1233 bdesc->next_blk = PFN_DOWN(addr);1234 }1235 1236 bdesc->sw_tag = sw_tag;1237 bdesc->block_sz = sg->length / 64;1238 }1239 }1240 1241 /*1242 * Make the above writes globally visible before tracing is1243 * enabled to make sure hardware sees them coherently.1244 */1245 wmb();1246}1247 1248static void msc_buffer_multi_free(struct msc *msc)1249{1250 struct msc_window *win, *iter;1251 1252 list_for_each_entry_safe(win, iter, &msc->win_list, entry)1253 msc_buffer_win_free(msc, win);1254}1255 1256static int msc_buffer_multi_alloc(struct msc *msc, unsigned long *nr_pages,1257 unsigned int nr_wins)1258{1259 int ret, i;1260 1261 for (i = 0; i < nr_wins; i++) {1262 ret = msc_buffer_win_alloc(msc, nr_pages[i]);1263 if (ret) {1264 msc_buffer_multi_free(msc);1265 return ret;1266 }1267 }1268 1269 msc_buffer_relink(msc);1270 1271 return 0;1272}1273 1274/**1275 * msc_buffer_free() - free buffers for MSC1276 * @msc: MSC device1277 *1278 * Free MSC's storage buffers.1279 *1280 * This modifies msc::win_list and msc::base, which requires msc::buf_mutex to1281 * serialize, so the caller is expected to hold it.1282 */1283static void msc_buffer_free(struct msc *msc)1284{1285 msc_buffer_set_wb(msc);1286 1287 if (msc->mode == MSC_MODE_SINGLE)1288 msc_buffer_contig_free(msc);1289 else if (msc->mode == MSC_MODE_MULTI)1290 msc_buffer_multi_free(msc);1291}1292 1293/**1294 * msc_buffer_alloc() - allocate a buffer for MSC1295 * @msc: MSC device1296 * @nr_pages: number of pages for each window1297 * @nr_wins: number of windows1298 *1299 * Allocate a storage buffer for MSC, depending on the msc::mode, it will be1300 * either done via msc_buffer_contig_alloc() for SINGLE operation mode or1301 * msc_buffer_win_alloc() for multiblock operation. The latter allocates one1302 * window per invocation, so in multiblock mode this can be called multiple1303 * times for the same MSC to allocate multiple windows.1304 *1305 * This modifies msc::win_list and msc::base, which requires msc::buf_mutex1306 * to serialize, so the caller is expected to hold it.1307 *1308 * Return: 0 on success, -errno otherwise.1309 */1310static int msc_buffer_alloc(struct msc *msc, unsigned long *nr_pages,1311 unsigned int nr_wins)1312{1313 int ret;1314 1315 /* -1: buffer not allocated */1316 if (atomic_read(&msc->user_count) != -1)1317 return -EBUSY;1318 1319 if (msc->mode == MSC_MODE_SINGLE) {1320 if (nr_wins != 1)1321 return -EINVAL;1322 1323 ret = msc_buffer_contig_alloc(msc, nr_pages[0] << PAGE_SHIFT);1324 } else if (msc->mode == MSC_MODE_MULTI) {1325 ret = msc_buffer_multi_alloc(msc, nr_pages, nr_wins);1326 } else {1327 ret = -EINVAL;1328 }1329 1330 if (!ret) {1331 msc_buffer_set_uc(msc);1332 1333 /* allocation should be visible before the counter goes to 0 */1334 smp_mb__before_atomic();1335 1336 if (WARN_ON_ONCE(atomic_cmpxchg(&msc->user_count, -1, 0) != -1))1337 return -EINVAL;1338 }1339 1340 return ret;1341}1342 1343/**1344 * msc_buffer_unlocked_free_unless_used() - free a buffer unless it's in use1345 * @msc: MSC device1346 *1347 * This will free MSC buffer unless it is in use or there is no allocated1348 * buffer.1349 * Caller needs to hold msc::buf_mutex.1350 *1351 * Return: 0 on successful deallocation or if there was no buffer to1352 * deallocate, -EBUSY if there are active users.1353 */1354static int msc_buffer_unlocked_free_unless_used(struct msc *msc)1355{1356 int count, ret = 0;1357 1358 count = atomic_cmpxchg(&msc->user_count, 0, -1);1359 1360 /* > 0: buffer is allocated and has users */1361 if (count > 0)1362 ret = -EBUSY;1363 /* 0: buffer is allocated, no users */1364 else if (!count)1365 msc_buffer_free(msc);1366 /* < 0: no buffer, nothing to do */1367 1368 return ret;1369}1370 1371/**1372 * msc_buffer_free_unless_used() - free a buffer unless it's in use1373 * @msc: MSC device1374 *1375 * This is a locked version of msc_buffer_unlocked_free_unless_used().1376 *1377 * Return: 0 on successful deallocation or if there was no buffer to1378 * deallocate, -EBUSY if there are active users.1379 */1380static int msc_buffer_free_unless_used(struct msc *msc)1381{1382 int ret;1383 1384 mutex_lock(&msc->buf_mutex);1385 ret = msc_buffer_unlocked_free_unless_used(msc);1386 mutex_unlock(&msc->buf_mutex);1387 1388 return ret;1389}1390 1391/**1392 * msc_buffer_get_page() - get MSC buffer page at a given offset1393 * @msc: MSC device1394 * @pgoff: page offset into the storage buffer1395 *1396 * This traverses msc::win_list, so holding msc::buf_mutex is expected from1397 * the caller.1398 *1399 * Return: page if @pgoff corresponds to a valid buffer page or NULL.1400 */1401static struct page *msc_buffer_get_page(struct msc *msc, unsigned long pgoff)1402{1403 struct msc_window *win;1404 struct scatterlist *sg;1405 unsigned int blk;1406 1407 if (msc->mode == MSC_MODE_SINGLE)1408 return msc_buffer_contig_get_page(msc, pgoff);1409 1410 list_for_each_entry(win, &msc->win_list, entry)1411 if (pgoff >= win->pgoff && pgoff < win->pgoff + win->nr_blocks)1412 goto found;1413 1414 return NULL;1415 1416found:1417 pgoff -= win->pgoff;1418 1419 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {1420 struct page *page = msc_sg_page(sg);1421 size_t pgsz = PFN_DOWN(sg->length);1422 1423 if (pgoff < pgsz)1424 return page + pgoff;1425 1426 pgoff -= pgsz;1427 }1428 1429 return NULL;1430}1431 1432/**1433 * struct msc_win_to_user_struct - data for copy_to_user() callback1434 * @buf: userspace buffer to copy data to1435 * @offset: running offset1436 */1437struct msc_win_to_user_struct {1438 char __user *buf;1439 unsigned long offset;1440};1441 1442/**1443 * msc_win_to_user() - iterator for msc_buffer_iterate() to copy data to user1444 * @data: callback's private data1445 * @src: source buffer1446 * @len: amount of data to copy from the source buffer1447 *1448 * Return: >= %0 for success or -errno for error.1449 */1450static unsigned long msc_win_to_user(void *data, void *src, size_t len)1451{1452 struct msc_win_to_user_struct *u = data;1453 unsigned long ret;1454 1455 ret = copy_to_user(u->buf + u->offset, src, len);1456 u->offset += len - ret;1457 1458 return ret;1459}1460 1461 1462/*1463 * file operations' callbacks1464 */1465 1466static int intel_th_msc_open(struct inode *inode, struct file *file)1467{1468 struct intel_th_device *thdev = file->private_data;1469 struct msc *msc = dev_get_drvdata(&thdev->dev);1470 struct msc_iter *iter;1471 1472 if (!capable(CAP_SYS_RAWIO))1473 return -EPERM;1474 1475 iter = msc_iter_install(msc);1476 if (IS_ERR(iter))1477 return PTR_ERR(iter);1478 1479 file->private_data = iter;1480 1481 return nonseekable_open(inode, file);1482}1483 1484static int intel_th_msc_release(struct inode *inode, struct file *file)1485{1486 struct msc_iter *iter = file->private_data;1487 struct msc *msc = iter->msc;1488 1489 msc_iter_remove(iter, msc);1490 1491 return 0;1492}1493 1494static ssize_t1495msc_single_to_user(struct msc *msc, char __user *buf, loff_t off, size_t len)1496{1497 unsigned long size = msc->nr_pages << PAGE_SHIFT, rem = len;1498 unsigned long start = off, tocopy = 0;1499 1500 if (msc->single_wrap) {1501 start += msc->single_sz;1502 if (start < size) {1503 tocopy = min(rem, size - start);1504 if (copy_to_user(buf, msc->base + start, tocopy))1505 return -EFAULT;1506 1507 buf += tocopy;1508 rem -= tocopy;1509 start += tocopy;1510 }1511 1512 start &= size - 1;1513 if (rem) {1514 tocopy = min(rem, msc->single_sz - start);1515 if (copy_to_user(buf, msc->base + start, tocopy))1516 return -EFAULT;1517 1518 rem -= tocopy;1519 }1520 1521 return len - rem;1522 }1523 1524 if (copy_to_user(buf, msc->base + start, rem))1525 return -EFAULT;1526 1527 return len;1528}1529 1530static ssize_t intel_th_msc_read(struct file *file, char __user *buf,1531 size_t len, loff_t *ppos)1532{1533 struct msc_iter *iter = file->private_data;1534 struct msc *msc = iter->msc;1535 size_t size;1536 loff_t off = *ppos;1537 ssize_t ret = 0;1538 1539 if (!atomic_inc_unless_negative(&msc->user_count))1540 return 0;1541 1542 if (msc->mode == MSC_MODE_SINGLE && !msc->single_wrap)1543 size = msc->single_sz;1544 else1545 size = msc->nr_pages << PAGE_SHIFT;1546 1547 if (!size)1548 goto put_count;1549 1550 if (off >= size)1551 goto put_count;1552 1553 if (off + len >= size)1554 len = size - off;1555 1556 if (msc->mode == MSC_MODE_SINGLE) {1557 ret = msc_single_to_user(msc, buf, off, len);1558 if (ret >= 0)1559 *ppos += ret;1560 } else if (msc->mode == MSC_MODE_MULTI) {1561 struct msc_win_to_user_struct u = {1562 .buf = buf,1563 .offset = 0,1564 };1565 1566 ret = msc_buffer_iterate(iter, len, &u, msc_win_to_user);1567 if (ret >= 0)1568 *ppos = iter->offset;1569 } else {1570 ret = -EINVAL;1571 }1572 1573put_count:1574 atomic_dec(&msc->user_count);1575 1576 return ret;1577}1578 1579/*1580 * vm operations callbacks (vm_ops)1581 */1582 1583static void msc_mmap_open(struct vm_area_struct *vma)1584{1585 struct msc_iter *iter = vma->vm_file->private_data;1586 struct msc *msc = iter->msc;1587 1588 atomic_inc(&msc->mmap_count);1589}1590 1591static void msc_mmap_close(struct vm_area_struct *vma)1592{1593 struct msc_iter *iter = vma->vm_file->private_data;1594 struct msc *msc = iter->msc;1595 unsigned long pg;1596 1597 if (!atomic_dec_and_mutex_lock(&msc->mmap_count, &msc->buf_mutex))1598 return;1599 1600 /* drop page _refcounts */1601 for (pg = 0; pg < msc->nr_pages; pg++) {1602 struct page *page = msc_buffer_get_page(msc, pg);1603 1604 if (WARN_ON_ONCE(!page))1605 continue;1606 1607 if (page->mapping)1608 page->mapping = NULL;1609 }1610 1611 /* last mapping -- drop user_count */1612 atomic_dec(&msc->user_count);1613 mutex_unlock(&msc->buf_mutex);1614}1615 1616static vm_fault_t msc_mmap_fault(struct vm_fault *vmf)1617{1618 struct msc_iter *iter = vmf->vma->vm_file->private_data;1619 struct msc *msc = iter->msc;1620 1621 vmf->page = msc_buffer_get_page(msc, vmf->pgoff);1622 if (!vmf->page)1623 return VM_FAULT_SIGBUS;1624 1625 get_page(vmf->page);1626 vmf->page->mapping = vmf->vma->vm_file->f_mapping;1627 vmf->page->index = vmf->pgoff;1628 1629 return 0;1630}1631 1632static const struct vm_operations_struct msc_mmap_ops = {1633 .open = msc_mmap_open,1634 .close = msc_mmap_close,1635 .fault = msc_mmap_fault,1636};1637 1638static int intel_th_msc_mmap(struct file *file, struct vm_area_struct *vma)1639{1640 unsigned long size = vma->vm_end - vma->vm_start;1641 struct msc_iter *iter = vma->vm_file->private_data;1642 struct msc *msc = iter->msc;1643 int ret = -EINVAL;1644 1645 if (!size || offset_in_page(size))1646 return -EINVAL;1647 1648 if (vma->vm_pgoff)1649 return -EINVAL;1650 1651 /* grab user_count once per mmap; drop in msc_mmap_close() */1652 if (!atomic_inc_unless_negative(&msc->user_count))1653 return -EINVAL;1654 1655 if (msc->mode != MSC_MODE_SINGLE &&1656 msc->mode != MSC_MODE_MULTI)1657 goto out;1658 1659 if (size >> PAGE_SHIFT != msc->nr_pages)1660 goto out;1661 1662 atomic_set(&msc->mmap_count, 1);1663 ret = 0;1664 1665out:1666 if (ret)1667 atomic_dec(&msc->user_count);1668 1669 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);1670 vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY);1671 vma->vm_ops = &msc_mmap_ops;1672 return ret;1673}1674 1675static const struct file_operations intel_th_msc_fops = {1676 .open = intel_th_msc_open,1677 .release = intel_th_msc_release,1678 .read = intel_th_msc_read,1679 .mmap = intel_th_msc_mmap,1680 .owner = THIS_MODULE,1681};1682 1683static void intel_th_msc_wait_empty(struct intel_th_device *thdev)1684{1685 struct msc *msc = dev_get_drvdata(&thdev->dev);1686 unsigned long count;1687 u32 reg;1688 1689 for (reg = 0, count = MSC_PLE_WAITLOOP_DEPTH;1690 count && !(reg & MSCSTS_PLE); count--) {1691 reg = __raw_readl(msc->reg_base + REG_MSU_MSC0STS);1692 cpu_relax();1693 }1694 1695 if (!count)1696 dev_dbg(msc_dev(msc), "timeout waiting for MSC0 PLE\n");1697}1698 1699static int intel_th_msc_init(struct msc *msc)1700{1701 atomic_set(&msc->user_count, -1);1702 1703 msc->mode = msc->multi_is_broken ? MSC_MODE_SINGLE : MSC_MODE_MULTI;1704 mutex_init(&msc->buf_mutex);1705 INIT_LIST_HEAD(&msc->win_list);1706 INIT_LIST_HEAD(&msc->iter_list);1707 1708 msc->burst_len =1709 (ioread32(msc->reg_base + REG_MSU_MSC0CTL) & MSC_LEN) >>1710 __ffs(MSC_LEN);1711 1712 return 0;1713}1714 1715static int msc_win_switch(struct msc *msc)1716{1717 struct msc_window *first;1718 1719 if (list_empty(&msc->win_list))1720 return -EINVAL;1721 1722 first = list_first_entry(&msc->win_list, struct msc_window, entry);1723 1724 if (msc_is_last_win(msc->cur_win))1725 msc->cur_win = first;1726 else1727 msc->cur_win = list_next_entry(msc->cur_win, entry);1728 1729 msc->base = msc_win_base(msc->cur_win);1730 msc->base_addr = msc_win_base_dma(msc->cur_win);1731 1732 intel_th_trace_switch(msc->thdev);1733 1734 return 0;1735}1736 1737/**1738 * intel_th_msc_window_unlock - put the window back in rotation1739 * @dev: MSC device to which this relates1740 * @sgt: buffer's sg_table for the window, does nothing if NULL1741 */1742void intel_th_msc_window_unlock(struct device *dev, struct sg_table *sgt)1743{1744 struct msc *msc = dev_get_drvdata(dev);1745 struct msc_window *win;1746 1747 if (!sgt)1748 return;1749 1750 win = msc_find_window(msc, sgt, false);1751 if (!win)1752 return;1753 1754 msc_win_set_lockout(win, WIN_LOCKED, WIN_READY);1755 if (msc->switch_on_unlock == win) {1756 msc->switch_on_unlock = NULL;1757 msc_win_switch(msc);1758 }1759}1760EXPORT_SYMBOL_GPL(intel_th_msc_window_unlock);1761 1762static void msc_work(struct work_struct *work)1763{1764 struct msc *msc = container_of(work, struct msc, work);1765 1766 intel_th_msc_deactivate(msc->thdev);1767}1768 1769static irqreturn_t intel_th_msc_interrupt(struct intel_th_device *thdev)1770{1771 struct msc *msc = dev_get_drvdata(&thdev->dev);1772 u32 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);1773 u32 mask = msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;1774 struct msc_window *win, *next_win;1775 1776 if (!msc->do_irq || !msc->mbuf)1777 return IRQ_NONE;1778 1779 msusts &= mask;1780 1781 if (!msusts)1782 return msc->enabled ? IRQ_HANDLED : IRQ_NONE;1783 1784 iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);1785 1786 if (!msc->enabled)1787 return IRQ_NONE;1788 1789 /* grab the window before we do the switch */1790 win = msc->cur_win;1791 if (!win)1792 return IRQ_HANDLED;1793 next_win = msc_next_window(win);1794 if (!next_win)1795 return IRQ_HANDLED;1796 1797 /* next window: if READY, proceed, if LOCKED, stop the trace */1798 if (msc_win_set_lockout(next_win, WIN_READY, WIN_INUSE)) {1799 if (msc->stop_on_full)1800 schedule_work(&msc->work);1801 else1802 msc->switch_on_unlock = next_win;1803 1804 return IRQ_HANDLED;1805 }1806 1807 /* current window: INUSE -> LOCKED */1808 msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);1809 1810 msc_win_switch(msc);1811 1812 if (msc->mbuf && msc->mbuf->ready)1813 msc->mbuf->ready(msc->mbuf_priv, win->sgt,1814 msc_win_total_sz(win));1815 1816 return IRQ_HANDLED;1817}1818 1819static const char * const msc_mode[] = {1820 [MSC_MODE_SINGLE] = "single",1821 [MSC_MODE_MULTI] = "multi",1822 [MSC_MODE_EXI] = "ExI",1823 [MSC_MODE_DEBUG] = "debug",1824};1825 1826static ssize_t1827wrap_show(struct device *dev, struct device_attribute *attr, char *buf)1828{1829 struct msc *msc = dev_get_drvdata(dev);1830 1831 return scnprintf(buf, PAGE_SIZE, "%d\n", msc->wrap);1832}1833 1834static ssize_t1835wrap_store(struct device *dev, struct device_attribute *attr, const char *buf,1836 size_t size)1837{1838 struct msc *msc = dev_get_drvdata(dev);1839 unsigned long val;1840 int ret;1841 1842 ret = kstrtoul(buf, 10, &val);1843 if (ret)1844 return ret;1845 1846 msc->wrap = !!val;1847 1848 return size;1849}1850 1851static DEVICE_ATTR_RW(wrap);1852 1853static void msc_buffer_unassign(struct msc *msc)1854{1855 lockdep_assert_held(&msc->buf_mutex);1856 1857 if (!msc->mbuf)1858 return;1859 1860 msc->mbuf->unassign(msc->mbuf_priv);1861 msu_buffer_put(msc->mbuf);1862 msc->mbuf_priv = NULL;1863 msc->mbuf = NULL;1864}1865 1866static ssize_t1867mode_show(struct device *dev, struct device_attribute *attr, char *buf)1868{1869 struct msc *msc = dev_get_drvdata(dev);1870 const char *mode = msc_mode[msc->mode];1871 ssize_t ret;1872 1873 mutex_lock(&msc->buf_mutex);1874 if (msc->mbuf)1875 mode = msc->mbuf->name;1876 ret = scnprintf(buf, PAGE_SIZE, "%s\n", mode);1877 mutex_unlock(&msc->buf_mutex);1878 1879 return ret;1880}1881 1882static ssize_t1883mode_store(struct device *dev, struct device_attribute *attr, const char *buf,1884 size_t size)1885{1886 const struct msu_buffer *mbuf = NULL;1887 struct msc *msc = dev_get_drvdata(dev);1888 size_t len = size;1889 char *cp, *mode;1890 int i, ret;1891 1892 if (!capable(CAP_SYS_RAWIO))1893 return -EPERM;1894 1895 cp = memchr(buf, '\n', len);1896 if (cp)1897 len = cp - buf;1898 1899 mode = kstrndup(buf, len, GFP_KERNEL);1900 if (!mode)1901 return -ENOMEM;1902 1903 i = match_string(msc_mode, ARRAY_SIZE(msc_mode), mode);1904 if (i >= 0) {1905 kfree(mode);1906 goto found;1907 }1908 1909 /* Buffer sinks only work with a usable IRQ */1910 if (!msc->do_irq) {1911 kfree(mode);1912 return -EINVAL;1913 }1914 1915 mbuf = msu_buffer_get(mode);1916 kfree(mode);1917 if (mbuf)1918 goto found;1919 1920 return -EINVAL;1921 1922found:1923 if (i == MSC_MODE_MULTI && msc->multi_is_broken)1924 return -EOPNOTSUPP;1925 1926 mutex_lock(&msc->buf_mutex);1927 ret = 0;1928 1929 /* Same buffer: do nothing */1930 if (mbuf && mbuf == msc->mbuf) {1931 /* put the extra reference we just got */1932 msu_buffer_put(mbuf);1933 goto unlock;1934 }1935 1936 ret = msc_buffer_unlocked_free_unless_used(msc);1937 if (ret)1938 goto unlock;1939 1940 if (mbuf) {1941 void *mbuf_priv = mbuf->assign(dev, &i);1942 1943 if (!mbuf_priv) {1944 ret = -ENOMEM;1945 goto unlock;1946 }1947 1948 msc_buffer_unassign(msc);1949 msc->mbuf_priv = mbuf_priv;1950 msc->mbuf = mbuf;1951 } else {1952 msc_buffer_unassign(msc);1953 }1954 1955 msc->mode = i;1956 1957unlock:1958 if (ret && mbuf)1959 msu_buffer_put(mbuf);1960 mutex_unlock(&msc->buf_mutex);1961 1962 return ret ? ret : size;1963}1964 1965static DEVICE_ATTR_RW(mode);1966 1967static ssize_t1968nr_pages_show(struct device *dev, struct device_attribute *attr, char *buf)1969{1970 struct msc *msc = dev_get_drvdata(dev);1971 struct msc_window *win;1972 size_t count = 0;1973 1974 mutex_lock(&msc->buf_mutex);1975 1976 if (msc->mode == MSC_MODE_SINGLE)1977 count = scnprintf(buf, PAGE_SIZE, "%ld\n", msc->nr_pages);1978 else if (msc->mode == MSC_MODE_MULTI) {1979 list_for_each_entry(win, &msc->win_list, entry) {1980 count += scnprintf(buf + count, PAGE_SIZE - count,1981 "%d%c", win->nr_blocks,1982 msc_is_last_win(win) ? '\n' : ',');1983 }1984 } else {1985 count = scnprintf(buf, PAGE_SIZE, "unsupported\n");1986 }1987 1988 mutex_unlock(&msc->buf_mutex);1989 1990 return count;1991}1992 1993static ssize_t1994nr_pages_store(struct device *dev, struct device_attribute *attr,1995 const char *buf, size_t size)1996{1997 struct msc *msc = dev_get_drvdata(dev);1998 unsigned long val, *win = NULL, *rewin;1999 size_t len = size;2000 const char *p = buf;2001 char *end, *s;2002 int ret, nr_wins = 0;2003 2004 if (!capable(CAP_SYS_RAWIO))2005 return -EPERM;2006 2007 ret = msc_buffer_free_unless_used(msc);2008 if (ret)2009 return ret;2010 2011 /* scan the comma-separated list of allocation sizes */2012 end = memchr(buf, '\n', len);2013 if (end)2014 len = end - buf;2015 2016 do {2017 end = memchr(p, ',', len);2018 s = kstrndup(p, end ? end - p : len, GFP_KERNEL);2019 if (!s) {2020 ret = -ENOMEM;2021 goto free_win;2022 }2023 2024 ret = kstrtoul(s, 10, &val);2025 kfree(s);2026 2027 if (ret || !val)2028 goto free_win;2029 2030 if (nr_wins && msc->mode == MSC_MODE_SINGLE) {2031 ret = -EINVAL;2032 goto free_win;2033 }2034 2035 nr_wins++;2036 rewin = krealloc_array(win, nr_wins, sizeof(*win), GFP_KERNEL);2037 if (!rewin) {2038 kfree(win);2039 return -ENOMEM;2040 }2041 2042 win = rewin;2043 win[nr_wins - 1] = val;2044 2045 if (!end)2046 break;2047 2048 /* consume the number and the following comma, hence +1 */2049 len -= end - p + 1;2050 p = end + 1;2051 } while (len);2052 2053 mutex_lock(&msc->buf_mutex);2054 ret = msc_buffer_alloc(msc, win, nr_wins);2055 mutex_unlock(&msc->buf_mutex);2056 2057free_win:2058 kfree(win);2059 2060 return ret ? ret : size;2061}2062 2063static DEVICE_ATTR_RW(nr_pages);2064 2065static ssize_t2066win_switch_store(struct device *dev, struct device_attribute *attr,2067 const char *buf, size_t size)2068{2069 struct msc *msc = dev_get_drvdata(dev);2070 unsigned long val;2071 int ret;2072 2073 ret = kstrtoul(buf, 10, &val);2074 if (ret)2075 return ret;2076 2077 if (val != 1)2078 return -EINVAL;2079 2080 ret = -EINVAL;2081 mutex_lock(&msc->buf_mutex);2082 /*2083 * Window switch can only happen in the "multi" mode.2084 * If a external buffer is engaged, they have the full2085 * control over window switching.2086 */2087 if (msc->mode == MSC_MODE_MULTI && !msc->mbuf)2088 ret = msc_win_switch(msc);2089 mutex_unlock(&msc->buf_mutex);2090 2091 return ret ? ret : size;2092}2093 2094static DEVICE_ATTR_WO(win_switch);2095 2096static ssize_t stop_on_full_show(struct device *dev,2097 struct device_attribute *attr, char *buf)2098{2099 struct msc *msc = dev_get_drvdata(dev);2100 2101 return sprintf(buf, "%d\n", msc->stop_on_full);2102}2103 2104static ssize_t stop_on_full_store(struct device *dev,2105 struct device_attribute *attr,2106 const char *buf, size_t size)2107{2108 struct msc *msc = dev_get_drvdata(dev);2109 int ret;2110 2111 ret = kstrtobool(buf, &msc->stop_on_full);2112 if (ret)2113 return ret;2114 2115 return size;2116}2117 2118static DEVICE_ATTR_RW(stop_on_full);2119 2120static struct attribute *msc_output_attrs[] = {2121 &dev_attr_wrap.attr,2122 &dev_attr_mode.attr,2123 &dev_attr_nr_pages.attr,2124 &dev_attr_win_switch.attr,2125 &dev_attr_stop_on_full.attr,2126 NULL,2127};2128 2129static const struct attribute_group msc_output_group = {2130 .attrs = msc_output_attrs,2131};2132 2133static int intel_th_msc_probe(struct intel_th_device *thdev)2134{2135 struct device *dev = &thdev->dev;2136 struct resource *res;2137 struct msc *msc;2138 void __iomem *base;2139 int err;2140 2141 res = intel_th_device_get_resource(thdev, IORESOURCE_MEM, 0);2142 if (!res)2143 return -ENODEV;2144 2145 base = devm_ioremap(dev, res->start, resource_size(res));2146 if (!base)2147 return -ENOMEM;2148 2149 msc = devm_kzalloc(dev, sizeof(*msc), GFP_KERNEL);2150 if (!msc)2151 return -ENOMEM;2152 2153 res = intel_th_device_get_resource(thdev, IORESOURCE_IRQ, 1);2154 if (!res)2155 msc->do_irq = 1;2156 2157 if (INTEL_TH_CAP(to_intel_th(thdev), multi_is_broken))2158 msc->multi_is_broken = 1;2159 2160 msc->index = thdev->id;2161 2162 msc->thdev = thdev;2163 msc->reg_base = base + msc->index * 0x100;2164 msc->msu_base = base;2165 2166 INIT_WORK(&msc->work, msc_work);2167 err = intel_th_msc_init(msc);2168 if (err)2169 return err;2170 2171 dev_set_drvdata(dev, msc);2172 2173 return 0;2174}2175 2176static void intel_th_msc_remove(struct intel_th_device *thdev)2177{2178 struct msc *msc = dev_get_drvdata(&thdev->dev);2179 int ret;2180 2181 intel_th_msc_deactivate(thdev);2182 2183 /*2184 * Buffers should not be used at this point except if the2185 * output character device is still open and the parent2186 * device gets detached from its bus, which is a FIXME.2187 */2188 ret = msc_buffer_free_unless_used(msc);2189 WARN_ON_ONCE(ret);2190}2191 2192static struct intel_th_driver intel_th_msc_driver = {2193 .probe = intel_th_msc_probe,2194 .remove = intel_th_msc_remove,2195 .irq = intel_th_msc_interrupt,2196 .wait_empty = intel_th_msc_wait_empty,2197 .activate = intel_th_msc_activate,2198 .deactivate = intel_th_msc_deactivate,2199 .fops = &intel_th_msc_fops,2200 .attr_group = &msc_output_group,2201 .driver = {2202 .name = "msc",2203 .owner = THIS_MODULE,2204 },2205};2206 2207module_driver(intel_th_msc_driver,2208 intel_th_driver_register,2209 intel_th_driver_unregister);2210 2211MODULE_LICENSE("GPL v2");2212MODULE_DESCRIPTION("Intel(R) Trace Hub Memory Storage Unit driver");2213MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>");2214