816 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */3#ifndef _IDXD_H_4#define _IDXD_H_5 6#include <linux/sbitmap.h>7#include <linux/dmaengine.h>8#include <linux/percpu-rwsem.h>9#include <linux/wait.h>10#include <linux/cdev.h>11#include <linux/idr.h>12#include <linux/pci.h>13#include <linux/bitmap.h>14#include <linux/perf_event.h>15#include <linux/iommu.h>16#include <linux/crypto.h>17#include <uapi/linux/idxd.h>18#include "registers.h"19 20#define IDXD_DRIVER_VERSION "1.00"21 22extern struct kmem_cache *idxd_desc_pool;23extern bool tc_override;24 25struct idxd_wq;26struct idxd_dev;27 28enum idxd_dev_type {29 IDXD_DEV_NONE = -1,30 IDXD_DEV_DSA = 0,31 IDXD_DEV_IAX,32 IDXD_DEV_WQ,33 IDXD_DEV_GROUP,34 IDXD_DEV_ENGINE,35 IDXD_DEV_CDEV,36 IDXD_DEV_CDEV_FILE,37 IDXD_DEV_MAX_TYPE,38};39 40struct idxd_dev {41 struct device conf_dev;42 enum idxd_dev_type type;43};44 45#define IDXD_REG_TIMEOUT 5046#define IDXD_DRAIN_TIMEOUT 500047 48enum idxd_type {49 IDXD_TYPE_UNKNOWN = -1,50 IDXD_TYPE_DSA = 0,51 IDXD_TYPE_IAX,52 IDXD_TYPE_MAX,53};54 55#define IDXD_NAME_SIZE 12856#define IDXD_PMU_EVENT_MAX 6457 58#define IDXD_ENQCMDS_RETRIES 3259#define IDXD_ENQCMDS_MAX_RETRIES 6460 61enum idxd_complete_type {62 IDXD_COMPLETE_NORMAL = 0,63 IDXD_COMPLETE_ABORT,64 IDXD_COMPLETE_DEV_FAIL,65};66 67struct idxd_desc;68 69struct idxd_device_driver {70 const char *name;71 enum idxd_dev_type *type;72 int (*probe)(struct idxd_dev *idxd_dev);73 void (*remove)(struct idxd_dev *idxd_dev);74 void (*desc_complete)(struct idxd_desc *desc,75 enum idxd_complete_type comp_type,76 bool free_desc,77 void *ctx, u32 *status);78 struct device_driver drv;79};80 81extern struct idxd_device_driver dsa_drv;82extern struct idxd_device_driver idxd_drv;83extern struct idxd_device_driver idxd_dmaengine_drv;84extern struct idxd_device_driver idxd_user_drv;85 86#define INVALID_INT_HANDLE -187struct idxd_irq_entry {88 int id;89 int vector;90 struct llist_head pending_llist;91 struct list_head work_list;92 /*93 * Lock to protect access between irq thread process descriptor94 * and irq thread processing error descriptor.95 */96 spinlock_t list_lock;97 int int_handle;98 ioasid_t pasid;99};100 101struct idxd_group {102 struct idxd_dev idxd_dev;103 struct idxd_device *idxd;104 struct grpcfg grpcfg;105 int id;106 int num_engines;107 int num_wqs;108 bool use_rdbuf_limit;109 u8 rdbufs_allowed;110 u8 rdbufs_reserved;111 int tc_a;112 int tc_b;113 int desc_progress_limit;114 int batch_progress_limit;115};116 117struct idxd_pmu {118 struct idxd_device *idxd;119 120 struct perf_event *event_list[IDXD_PMU_EVENT_MAX];121 int n_events;122 123 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);124 125 struct pmu pmu;126 char name[IDXD_NAME_SIZE];127 128 int n_counters;129 int counter_width;130 int n_event_categories;131 132 bool per_counter_caps_supported;133 unsigned long supported_event_categories;134 135 unsigned long supported_filters;136 int n_filters;137};138 139#define IDXD_MAX_PRIORITY 0xf140 141enum {142 COUNTER_FAULTS = 0,143 COUNTER_FAULT_FAILS,144 COUNTER_MAX145};146 147enum idxd_wq_state {148 IDXD_WQ_DISABLED = 0,149 IDXD_WQ_ENABLED,150};151 152enum idxd_wq_flag {153 WQ_FLAG_DEDICATED = 0,154 WQ_FLAG_BLOCK_ON_FAULT,155 WQ_FLAG_ATS_DISABLE,156 WQ_FLAG_PRS_DISABLE,157};158 159enum idxd_wq_type {160 IDXD_WQT_NONE = 0,161 IDXD_WQT_KERNEL,162 IDXD_WQT_USER,163};164 165struct idxd_cdev {166 struct idxd_wq *wq;167 struct cdev cdev;168 struct idxd_dev idxd_dev;169 int minor;170};171 172#define DRIVER_NAME_SIZE 128173 174#define IDXD_ALLOCATED_BATCH_SIZE 128U175#define WQ_NAME_SIZE 1024176#define WQ_TYPE_SIZE 10177 178#define WQ_DEFAULT_QUEUE_DEPTH 16179#define WQ_DEFAULT_MAX_XFER SZ_2M180#define WQ_DEFAULT_MAX_BATCH 32181 182enum idxd_op_type {183 IDXD_OP_BLOCK = 0,184 IDXD_OP_NONBLOCK = 1,185};186 187struct idxd_dma_chan {188 struct dma_chan chan;189 struct idxd_wq *wq;190};191 192struct idxd_wq {193 void __iomem *portal;194 u32 portal_offset;195 unsigned int enqcmds_retries;196 struct percpu_ref wq_active;197 struct completion wq_dead;198 struct completion wq_resurrect;199 struct idxd_dev idxd_dev;200 struct idxd_cdev *idxd_cdev;201 struct wait_queue_head err_queue;202 struct workqueue_struct *wq;203 struct idxd_device *idxd;204 int id;205 struct idxd_irq_entry ie;206 enum idxd_wq_type type;207 struct idxd_group *group;208 int client_count;209 struct mutex wq_lock; /* mutex for workqueue */210 u32 size;211 u32 threshold;212 u32 priority;213 enum idxd_wq_state state;214 unsigned long flags;215 union wqcfg *wqcfg;216 unsigned long *opcap_bmap;217 218 struct dsa_hw_desc **hw_descs;219 int num_descs;220 union {221 struct dsa_completion_record *compls;222 struct iax_completion_record *iax_compls;223 };224 dma_addr_t compls_addr;225 int compls_size;226 struct idxd_desc **descs;227 struct sbitmap_queue sbq;228 struct idxd_dma_chan *idxd_chan;229 char name[WQ_NAME_SIZE + 1];230 u64 max_xfer_bytes;231 u32 max_batch_size;232 233 /* Lock to protect upasid_xa access. */234 struct mutex uc_lock;235 struct xarray upasid_xa;236 237 char driver_name[DRIVER_NAME_SIZE + 1];238};239 240struct idxd_engine {241 struct idxd_dev idxd_dev;242 int id;243 struct idxd_group *group;244 struct idxd_device *idxd;245};246 247/* shadow registers */248struct idxd_hw {249 u32 version;250 union gen_cap_reg gen_cap;251 union wq_cap_reg wq_cap;252 union group_cap_reg group_cap;253 union engine_cap_reg engine_cap;254 struct opcap opcap;255 u32 cmd_cap;256 union iaa_cap_reg iaa_cap;257};258 259enum idxd_device_state {260 IDXD_DEV_HALTED = -1,261 IDXD_DEV_DISABLED = 0,262 IDXD_DEV_ENABLED,263};264 265enum idxd_device_flag {266 IDXD_FLAG_CONFIGURABLE = 0,267 IDXD_FLAG_CMD_RUNNING,268 IDXD_FLAG_PASID_ENABLED,269 IDXD_FLAG_USER_PASID_ENABLED,270};271 272struct idxd_dma_dev {273 struct idxd_device *idxd;274 struct dma_device dma;275};276 277typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd);278 279struct idxd_driver_data {280 const char *name_prefix;281 enum idxd_type type;282 const struct device_type *dev_type;283 int compl_size;284 int align;285 int evl_cr_off;286 int cr_status_off;287 int cr_result_off;288 bool user_submission_safe;289 load_device_defaults_fn_t load_device_defaults;290};291 292struct idxd_evl {293 /* Lock to protect event log access. */294 struct mutex lock;295 void *log;296 dma_addr_t dma;297 /* Total size of event log = number of entries * entry size. */298 unsigned int log_size;299 /* The number of entries in the event log. */300 u16 size;301 unsigned long *bmap;302 bool batch_fail[IDXD_MAX_BATCH_IDENT];303};304 305struct idxd_evl_fault {306 struct work_struct work;307 struct idxd_wq *wq;308 u8 status;309 310 /* make this last member always */311 struct __evl_entry entry[];312};313 314struct idxd_device {315 struct idxd_dev idxd_dev;316 struct idxd_driver_data *data;317 struct list_head list;318 struct idxd_hw hw;319 enum idxd_device_state state;320 unsigned long flags;321 int id;322 int major;323 u32 cmd_status;324 struct idxd_irq_entry ie; /* misc irq, msix 0 */325 326 struct pci_dev *pdev;327 void __iomem *reg_base;328 329 spinlock_t dev_lock; /* spinlock for device */330 spinlock_t cmd_lock; /* spinlock for device commands */331 struct completion *cmd_done;332 struct idxd_group **groups;333 struct idxd_wq **wqs;334 struct idxd_engine **engines;335 336 struct iommu_sva *sva;337 unsigned int pasid;338 339 int num_groups;340 int irq_cnt;341 bool request_int_handles;342 343 u32 msix_perm_offset;344 u32 wqcfg_offset;345 u32 grpcfg_offset;346 u32 perfmon_offset;347 348 u64 max_xfer_bytes;349 u32 max_batch_size;350 int max_groups;351 int max_engines;352 int max_rdbufs;353 int max_wqs;354 int max_wq_size;355 int rdbuf_limit;356 int nr_rdbufs; /* non-reserved read buffers */357 unsigned int wqcfg_size;358 unsigned long *wq_enable_map;359 360 union sw_err_reg sw_err;361 wait_queue_head_t cmd_waitq;362 363 struct idxd_dma_dev *idxd_dma;364 struct workqueue_struct *wq;365 struct work_struct work;366 367 struct idxd_pmu *idxd_pmu;368 369 unsigned long *opcap_bmap;370 struct idxd_evl *evl;371 struct kmem_cache *evl_cache;372 373 struct dentry *dbgfs_dir;374 struct dentry *dbgfs_evl_file;375 376 bool user_submission_safe;377};378 379static inline unsigned int evl_ent_size(struct idxd_device *idxd)380{381 return idxd->hw.gen_cap.evl_support ?382 (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0;383}384 385static inline unsigned int evl_size(struct idxd_device *idxd)386{387 return idxd->evl->size * evl_ent_size(idxd);388}389 390struct crypto_ctx {391 struct acomp_req *req;392 struct crypto_tfm *tfm;393 dma_addr_t src_addr;394 dma_addr_t dst_addr;395 bool compress;396};397 398/* IDXD software descriptor */399struct idxd_desc {400 union {401 struct dsa_hw_desc *hw;402 struct iax_hw_desc *iax_hw;403 };404 dma_addr_t desc_dma;405 union {406 struct dsa_completion_record *completion;407 struct iax_completion_record *iax_completion;408 };409 dma_addr_t compl_dma;410 union {411 struct dma_async_tx_descriptor txd;412 struct crypto_ctx crypto;413 };414 struct llist_node llnode;415 struct list_head list;416 int id;417 int cpu;418 struct idxd_wq *wq;419};420 421/*422 * This is software defined error for the completion status. We overload the error code423 * that will never appear in completion status and only SWERR register.424 */425enum idxd_completion_status {426 IDXD_COMP_DESC_ABORT = 0xff,427};428 429#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev430#define wq_confdev(wq) &wq->idxd_dev.conf_dev431#define engine_confdev(engine) &engine->idxd_dev.conf_dev432#define group_confdev(group) &group->idxd_dev.conf_dev433#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev434#define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev)435 436#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)437#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)438#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)439 440static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq)441{442 struct device *dev = wq_confdev(wq);443 struct idxd_device_driver *idxd_drv =444 container_of(dev->driver, struct idxd_device_driver, drv);445 446 return idxd_drv;447}448 449static inline struct idxd_device *confdev_to_idxd(struct device *dev)450{451 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);452 453 return idxd_dev_to_idxd(idxd_dev);454}455 456static inline struct idxd_wq *confdev_to_wq(struct device *dev)457{458 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);459 460 return idxd_dev_to_wq(idxd_dev);461}462 463static inline struct idxd_engine *confdev_to_engine(struct device *dev)464{465 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);466 467 return container_of(idxd_dev, struct idxd_engine, idxd_dev);468}469 470static inline struct idxd_group *confdev_to_group(struct device *dev)471{472 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);473 474 return container_of(idxd_dev, struct idxd_group, idxd_dev);475}476 477static inline struct idxd_cdev *dev_to_cdev(struct device *dev)478{479 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);480 481 return container_of(idxd_dev, struct idxd_cdev, idxd_dev);482}483 484static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)485{486 if (type >= IDXD_DEV_MAX_TYPE) {487 idev->type = IDXD_DEV_NONE;488 return;489 }490 491 idev->type = type;492}493 494static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)495{496 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;497}498 499static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)500{501 return container_of(ie, struct idxd_wq, ie);502}503 504static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)505{506 return container_of(ie, struct idxd_device, ie);507}508 509static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable)510{511 union gencfg_reg reg;512 513 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);514 reg.user_int_en = enable;515 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);516}517 518extern const struct bus_type dsa_bus_type;519 520extern bool support_enqcmd;521extern struct ida idxd_ida;522extern const struct device_type dsa_device_type;523extern const struct device_type iax_device_type;524extern const struct device_type idxd_wq_device_type;525extern const struct device_type idxd_engine_device_type;526extern const struct device_type idxd_group_device_type;527 528static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)529{530 return idxd_dev->type == IDXD_DEV_DSA;531}532 533static inline bool is_iax_dev(struct idxd_dev *idxd_dev)534{535 return idxd_dev->type == IDXD_DEV_IAX;536}537 538static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)539{540 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);541}542 543static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)544{545 return idxd_dev->type == IDXD_DEV_WQ;546}547 548static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)549{550 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)551 return true;552 return false;553}554 555static inline bool is_idxd_wq_user(struct idxd_wq *wq)556{557 return wq->type == IDXD_WQT_USER;558}559 560static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)561{562 return wq->type == IDXD_WQT_KERNEL;563}564 565static inline bool wq_dedicated(struct idxd_wq *wq)566{567 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);568}569 570static inline bool wq_shared(struct idxd_wq *wq)571{572 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);573}574 575static inline bool device_pasid_enabled(struct idxd_device *idxd)576{577 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);578}579 580static inline bool device_user_pasid_enabled(struct idxd_device *idxd)581{582 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);583}584 585static inline bool wq_pasid_enabled(struct idxd_wq *wq)586{587 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||588 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));589}590 591static inline bool wq_shared_supported(struct idxd_wq *wq)592{593 return (support_enqcmd && wq_pasid_enabled(wq));594}595 596enum idxd_portal_prot {597 IDXD_PORTAL_UNLIMITED = 0,598 IDXD_PORTAL_LIMITED,599};600 601enum idxd_interrupt_type {602 IDXD_IRQ_MSIX = 0,603 IDXD_IRQ_IMS,604};605 606static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)607{608 return prot * 0x1000;609}610 611static inline int idxd_get_wq_portal_full_offset(int wq_id,612 enum idxd_portal_prot prot)613{614 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);615}616 617#define IDXD_PORTAL_MASK (PAGE_SIZE - 1)618 619/*620 * Even though this function can be accessed by multiple threads, it is safe to use.621 * At worst the address gets used more than once before it gets incremented. We don't622 * hit a threshold until iops becomes many million times a second. So the occasional623 * reuse of the same address is tolerable compare to using an atomic variable. This is624 * safe on a system that has atomic load/store for 32bit integers. Given that this is an625 * Intel iEP device, that should not be a problem.626 */627static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)628{629 int ofs = wq->portal_offset;630 631 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;632 return wq->portal + ofs;633}634 635static inline void idxd_wq_get(struct idxd_wq *wq)636{637 wq->client_count++;638}639 640static inline void idxd_wq_put(struct idxd_wq *wq)641{642 wq->client_count--;643}644 645static inline int idxd_wq_refcount(struct idxd_wq *wq)646{647 return wq->client_count;648};649 650static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private)651{652 dev_set_drvdata(wq_confdev(wq), private);653}654 655static inline void *idxd_wq_get_private(struct idxd_wq *wq)656{657 return dev_get_drvdata(wq_confdev(wq));658}659 660/*661 * Intel IAA does not support batch processing.662 * The max batch size of device, max batch size of wq and663 * max batch shift of wqcfg should be always 0 on IAA.664 */665static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd,666 u32 max_batch_size)667{668 if (idxd_type == IDXD_TYPE_IAX)669 idxd->max_batch_size = 0;670 else671 idxd->max_batch_size = max_batch_size;672}673 674static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq,675 u32 max_batch_size)676{677 if (idxd_type == IDXD_TYPE_IAX)678 wq->max_batch_size = 0;679 else680 wq->max_batch_size = max_batch_size;681}682 683static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg,684 u32 max_batch_shift)685{686 if (idxd_type == IDXD_TYPE_IAX)687 wqcfg->max_batch_shift = 0;688 else689 wqcfg->max_batch_shift = max_batch_shift;690}691 692static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev)693{694 return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0);695}696 697#define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*")698#define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d"699 700int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,701 struct module *module, const char *mod_name);702#define idxd_driver_register(driver) \703 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)704 705void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);706 707#define module_idxd_driver(__idxd_driver) \708 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)709 710void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);711void idxd_dma_complete_txd(struct idxd_desc *desc,712 enum idxd_complete_type comp_type,713 bool free_desc, void *ctx, u32 *status);714 715static inline void idxd_desc_complete(struct idxd_desc *desc,716 enum idxd_complete_type comp_type,717 bool free_desc)718{719 struct idxd_device_driver *drv;720 u32 status;721 722 drv = wq_to_idxd_drv(desc->wq);723 if (drv->desc_complete)724 drv->desc_complete(desc, comp_type, free_desc,725 &desc->txd, &status);726}727 728int idxd_register_bus_type(void);729void idxd_unregister_bus_type(void);730int idxd_register_devices(struct idxd_device *idxd);731void idxd_unregister_devices(struct idxd_device *idxd);732void idxd_wqs_quiesce(struct idxd_device *idxd);733bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);734void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count);735int idxd_load_iaa_device_defaults(struct idxd_device *idxd);736 737/* device interrupt control */738irqreturn_t idxd_misc_thread(int vec, void *data);739irqreturn_t idxd_wq_thread(int irq, void *data);740void idxd_mask_error_interrupts(struct idxd_device *idxd);741void idxd_unmask_error_interrupts(struct idxd_device *idxd);742 743/* device control */744int idxd_device_drv_probe(struct idxd_dev *idxd_dev);745void idxd_device_drv_remove(struct idxd_dev *idxd_dev);746int idxd_drv_enable_wq(struct idxd_wq *wq);747void idxd_drv_disable_wq(struct idxd_wq *wq);748int idxd_device_init_reset(struct idxd_device *idxd);749int idxd_device_enable(struct idxd_device *idxd);750int idxd_device_disable(struct idxd_device *idxd);751void idxd_device_reset(struct idxd_device *idxd);752void idxd_device_clear_state(struct idxd_device *idxd);753int idxd_device_config(struct idxd_device *idxd);754void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);755int idxd_device_load_config(struct idxd_device *idxd);756int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,757 enum idxd_interrupt_type irq_type);758int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,759 enum idxd_interrupt_type irq_type);760 761/* work queue control */762void idxd_wqs_unmap_portal(struct idxd_device *idxd);763int idxd_wq_alloc_resources(struct idxd_wq *wq);764void idxd_wq_free_resources(struct idxd_wq *wq);765int idxd_wq_enable(struct idxd_wq *wq);766int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);767void idxd_wq_drain(struct idxd_wq *wq);768void idxd_wq_reset(struct idxd_wq *wq);769int idxd_wq_map_portal(struct idxd_wq *wq);770void idxd_wq_unmap_portal(struct idxd_wq *wq);771int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);772int idxd_wq_disable_pasid(struct idxd_wq *wq);773void __idxd_wq_quiesce(struct idxd_wq *wq);774void idxd_wq_quiesce(struct idxd_wq *wq);775int idxd_wq_init_percpu_ref(struct idxd_wq *wq);776void idxd_wq_free_irq(struct idxd_wq *wq);777int idxd_wq_request_irq(struct idxd_wq *wq);778 779/* submission */780int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);781struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);782int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);783 784/* dmaengine */785int idxd_register_dma_device(struct idxd_device *idxd);786void idxd_unregister_dma_device(struct idxd_device *idxd);787 788/* cdev */789int idxd_cdev_register(void);790void idxd_cdev_remove(void);791int idxd_cdev_get_major(struct idxd_device *idxd);792int idxd_wq_add_cdev(struct idxd_wq *wq);793void idxd_wq_del_cdev(struct idxd_wq *wq);794int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,795 void *buf, int len);796void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index);797 798/* perfmon */799#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)800int perfmon_pmu_init(struct idxd_device *idxd);801void perfmon_pmu_remove(struct idxd_device *idxd);802void perfmon_counter_overflow(struct idxd_device *idxd);803#else804static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }805static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}806static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}807#endif808 809/* debugfs */810int idxd_device_init_debugfs(struct idxd_device *idxd);811void idxd_device_remove_debugfs(struct idxd_device *idxd);812int idxd_init_debugfs(void);813void idxd_remove_debugfs(void);814 815#endif816