498 lines · c
1/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */2/* Copyright(c) 2014 - 2020 Intel Corporation */3#ifndef ADF_ACCEL_DEVICES_H_4#define ADF_ACCEL_DEVICES_H_5#include <linux/interrupt.h>6#include <linux/module.h>7#include <linux/list.h>8#include <linux/io.h>9#include <linux/pci.h>10#include <linux/ratelimit.h>11#include <linux/types.h>12#include <linux/qat/qat_mig_dev.h>13#include "adf_cfg_common.h"14#include "adf_rl.h"15#include "adf_telemetry.h"16#include "adf_pfvf_msg.h"17#include "icp_qat_hw.h"18 19#define ADF_DH895XCC_DEVICE_NAME "dh895xcc"20#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf"21#define ADF_C62X_DEVICE_NAME "c6xx"22#define ADF_C62XVF_DEVICE_NAME "c6xxvf"23#define ADF_C3XXX_DEVICE_NAME "c3xxx"24#define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf"25#define ADF_4XXX_DEVICE_NAME "4xxx"26#define ADF_420XX_DEVICE_NAME "420xx"27#define ADF_4XXX_PCI_DEVICE_ID 0x494028#define ADF_4XXXIOV_PCI_DEVICE_ID 0x494129#define ADF_401XX_PCI_DEVICE_ID 0x494230#define ADF_401XXIOV_PCI_DEVICE_ID 0x494331#define ADF_402XX_PCI_DEVICE_ID 0x494432#define ADF_402XXIOV_PCI_DEVICE_ID 0x494533#define ADF_420XX_PCI_DEVICE_ID 0x494634#define ADF_420XXIOV_PCI_DEVICE_ID 0x494735#define ADF_DEVICE_FUSECTL_OFFSET 0x4036#define ADF_DEVICE_LEGFUSE_OFFSET 0x4C37#define ADF_DEVICE_FUSECTL_MASK 0x8000000038#define ADF_PCI_MAX_BARS 339#define ADF_DEVICE_NAME_LENGTH 3240#define ADF_ETR_MAX_RINGS_PER_BANK 1641#define ADF_MAX_MSIX_VECTOR_NAME 4842#define ADF_DEVICE_NAME_PREFIX "qat_"43 44enum adf_accel_capabilities {45 ADF_ACCEL_CAPABILITIES_NULL = 0,46 ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1,47 ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2,48 ADF_ACCEL_CAPABILITIES_CIPHER = 4,49 ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8,50 ADF_ACCEL_CAPABILITIES_COMPRESSION = 32,51 ADF_ACCEL_CAPABILITIES_LZS_COMPRESSION = 64,52 ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 12853};54 55struct adf_bar {56 resource_size_t base_addr;57 void __iomem *virt_addr;58 resource_size_t size;59};60 61struct adf_irq {62 bool enabled;63 char name[ADF_MAX_MSIX_VECTOR_NAME];64};65 66struct adf_accel_msix {67 struct adf_irq *irqs;68 u32 num_entries;69};70 71struct adf_accel_pci {72 struct pci_dev *pci_dev;73 struct adf_accel_msix msix_entries;74 struct adf_bar pci_bars[ADF_PCI_MAX_BARS];75 u8 revid;76 u8 sku;77};78 79enum dev_state {80 DEV_DOWN = 0,81 DEV_UP82};83 84enum dev_sku_info {85 DEV_SKU_1 = 0,86 DEV_SKU_2,87 DEV_SKU_3,88 DEV_SKU_4,89 DEV_SKU_VF,90 DEV_SKU_UNKNOWN,91};92 93enum ras_errors {94 ADF_RAS_CORR,95 ADF_RAS_UNCORR,96 ADF_RAS_FATAL,97 ADF_RAS_ERRORS,98};99 100struct adf_error_counters {101 atomic_t counter[ADF_RAS_ERRORS];102 bool sysfs_added;103 bool enabled;104};105 106static inline const char *get_sku_info(enum dev_sku_info info)107{108 switch (info) {109 case DEV_SKU_1:110 return "SKU1";111 case DEV_SKU_2:112 return "SKU2";113 case DEV_SKU_3:114 return "SKU3";115 case DEV_SKU_4:116 return "SKU4";117 case DEV_SKU_VF:118 return "SKUVF";119 case DEV_SKU_UNKNOWN:120 default:121 break;122 }123 return "Unknown SKU";124}125 126struct adf_hw_device_class {127 const char *name;128 const enum adf_device_type type;129 u32 instances;130};131 132struct arb_info {133 u32 arb_cfg;134 u32 arb_offset;135 u32 wt2sam_offset;136};137 138struct admin_info {139 u32 admin_msg_ur;140 u32 admin_msg_lr;141 u32 mailbox_offset;142};143 144struct ring_config {145 u64 base;146 u32 config;147 u32 head;148 u32 tail;149 u32 reserved0;150};151 152struct bank_state {153 u32 ringstat0;154 u32 ringstat1;155 u32 ringuostat;156 u32 ringestat;157 u32 ringnestat;158 u32 ringnfstat;159 u32 ringfstat;160 u32 ringcstat0;161 u32 ringcstat1;162 u32 ringcstat2;163 u32 ringcstat3;164 u32 iaintflagen;165 u32 iaintflagreg;166 u32 iaintflagsrcsel0;167 u32 iaintflagsrcsel1;168 u32 iaintcolen;169 u32 iaintcolctl;170 u32 iaintflagandcolen;171 u32 ringexpstat;172 u32 ringexpintenable;173 u32 ringsrvarben;174 u32 reserved0;175 struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK];176};177 178struct adf_hw_csr_ops {179 u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size);180 u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank,181 u32 ring);182 void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank,183 u32 ring, u32 value);184 u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank,185 u32 ring);186 void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank,187 u32 ring, u32 value);188 u32 (*read_csr_stat)(void __iomem *csr_base_addr, u32 bank);189 u32 (*read_csr_uo_stat)(void __iomem *csr_base_addr, u32 bank);190 u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank);191 u32 (*read_csr_ne_stat)(void __iomem *csr_base_addr, u32 bank);192 u32 (*read_csr_nf_stat)(void __iomem *csr_base_addr, u32 bank);193 u32 (*read_csr_f_stat)(void __iomem *csr_base_addr, u32 bank);194 u32 (*read_csr_c_stat)(void __iomem *csr_base_addr, u32 bank);195 u32 (*read_csr_exp_stat)(void __iomem *csr_base_addr, u32 bank);196 u32 (*read_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank);197 void (*write_csr_exp_int_en)(void __iomem *csr_base_addr, u32 bank,198 u32 value);199 u32 (*read_csr_ring_config)(void __iomem *csr_base_addr, u32 bank,200 u32 ring);201 void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank,202 u32 ring, u32 value);203 dma_addr_t (*read_csr_ring_base)(void __iomem *csr_base_addr, u32 bank,204 u32 ring);205 void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank,206 u32 ring, dma_addr_t addr);207 u32 (*read_csr_int_en)(void __iomem *csr_base_addr, u32 bank);208 void (*write_csr_int_en)(void __iomem *csr_base_addr, u32 bank,209 u32 value);210 u32 (*read_csr_int_flag)(void __iomem *csr_base_addr, u32 bank);211 void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank,212 u32 value);213 u32 (*read_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank);214 void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank);215 void (*write_csr_int_srcsel_w_val)(void __iomem *csr_base_addr,216 u32 bank, u32 value);217 u32 (*read_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank);218 void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank,219 u32 value);220 u32 (*read_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank);221 void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank,222 u32 value);223 u32 (*read_csr_int_flag_and_col)(void __iomem *csr_base_addr,224 u32 bank);225 void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr,226 u32 bank, u32 value);227 u32 (*read_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank);228 void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank,229 u32 value);230 u32 (*get_int_col_ctl_enable_mask)(void);231};232 233struct adf_cfg_device_data;234struct adf_accel_dev;235struct adf_etr_data;236struct adf_etr_ring_data;237 238struct adf_ras_ops {239 void (*enable_ras_errors)(struct adf_accel_dev *accel_dev);240 void (*disable_ras_errors)(struct adf_accel_dev *accel_dev);241 bool (*handle_interrupt)(struct adf_accel_dev *accel_dev,242 bool *reset_required);243};244 245struct adf_pfvf_ops {246 int (*enable_comms)(struct adf_accel_dev *accel_dev);247 u32 (*get_pf2vf_offset)(u32 i);248 u32 (*get_vf2pf_offset)(u32 i);249 void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);250 void (*disable_all_vf2pf_interrupts)(void __iomem *pmisc_addr);251 u32 (*disable_pending_vf2pf_interrupts)(void __iomem *pmisc_addr);252 int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg,253 u32 pfvf_offset, struct mutex *csr_lock);254 struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev,255 u32 pfvf_offset, u8 compat_ver);256};257 258struct adf_dc_ops {259 void (*build_deflate_ctx)(void *ctx);260};261 262struct qat_migdev_ops {263 int (*init)(struct qat_mig_dev *mdev);264 void (*cleanup)(struct qat_mig_dev *mdev);265 void (*reset)(struct qat_mig_dev *mdev);266 int (*open)(struct qat_mig_dev *mdev);267 void (*close)(struct qat_mig_dev *mdev);268 int (*suspend)(struct qat_mig_dev *mdev);269 int (*resume)(struct qat_mig_dev *mdev);270 int (*save_state)(struct qat_mig_dev *mdev);271 int (*save_setup)(struct qat_mig_dev *mdev);272 int (*load_state)(struct qat_mig_dev *mdev);273 int (*load_setup)(struct qat_mig_dev *mdev, int size);274};275 276struct adf_dev_err_mask {277 u32 cppagentcmdpar_mask;278 u32 parerr_ath_cph_mask;279 u32 parerr_cpr_xlt_mask;280 u32 parerr_dcpr_ucs_mask;281 u32 parerr_pke_mask;282 u32 parerr_wat_wcp_mask;283 u32 ssmfeatren_mask;284};285 286struct adf_hw_device_data {287 struct adf_hw_device_class *dev_class;288 u32 (*get_accel_mask)(struct adf_hw_device_data *self);289 u32 (*get_ae_mask)(struct adf_hw_device_data *self);290 u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev);291 u32 (*get_sram_bar_id)(struct adf_hw_device_data *self);292 u32 (*get_misc_bar_id)(struct adf_hw_device_data *self);293 u32 (*get_etr_bar_id)(struct adf_hw_device_data *self);294 u32 (*get_num_aes)(struct adf_hw_device_data *self);295 u32 (*get_num_accels)(struct adf_hw_device_data *self);296 void (*get_arb_info)(struct arb_info *arb_csrs_info);297 void (*get_admin_info)(struct admin_info *admin_csrs_info);298 enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self);299 u16 (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev);300 int (*alloc_irq)(struct adf_accel_dev *accel_dev);301 void (*free_irq)(struct adf_accel_dev *accel_dev);302 void (*enable_error_correction)(struct adf_accel_dev *accel_dev);303 int (*init_admin_comms)(struct adf_accel_dev *accel_dev);304 void (*exit_admin_comms)(struct adf_accel_dev *accel_dev);305 int (*send_admin_init)(struct adf_accel_dev *accel_dev);306 int (*start_timer)(struct adf_accel_dev *accel_dev);307 void (*stop_timer)(struct adf_accel_dev *accel_dev);308 void (*check_hb_ctrs)(struct adf_accel_dev *accel_dev);309 uint32_t (*get_hb_clock)(struct adf_hw_device_data *self);310 int (*measure_clock)(struct adf_accel_dev *accel_dev);311 int (*init_arb)(struct adf_accel_dev *accel_dev);312 void (*exit_arb)(struct adf_accel_dev *accel_dev);313 const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev);314 int (*init_device)(struct adf_accel_dev *accel_dev);315 int (*enable_pm)(struct adf_accel_dev *accel_dev);316 bool (*handle_pm_interrupt)(struct adf_accel_dev *accel_dev);317 void (*disable_iov)(struct adf_accel_dev *accel_dev);318 void (*configure_iov_threads)(struct adf_accel_dev *accel_dev,319 bool enable);320 void (*enable_ints)(struct adf_accel_dev *accel_dev);321 void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);322 int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr);323 int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number,324 struct bank_state *state);325 int (*bank_state_restore)(struct adf_accel_dev *accel_dev,326 u32 bank_number, struct bank_state *state);327 void (*reset_device)(struct adf_accel_dev *accel_dev);328 void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);329 const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num);330 u32 (*uof_get_num_objs)(struct adf_accel_dev *accel_dev);331 int (*uof_get_obj_type)(struct adf_accel_dev *accel_dev, u32 obj_num);332 u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num);333 int (*get_rp_group)(struct adf_accel_dev *accel_dev, u32 ae_mask);334 u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num);335 int (*dev_config)(struct adf_accel_dev *accel_dev);336 struct adf_pfvf_ops pfvf_ops;337 struct adf_hw_csr_ops csr_ops;338 struct adf_dc_ops dc_ops;339 struct adf_ras_ops ras_ops;340 struct adf_dev_err_mask dev_err_mask;341 struct adf_rl_hw_data rl_data;342 struct adf_tl_hw_data tl_data;343 struct qat_migdev_ops vfmig_ops;344 const char *fw_name;345 const char *fw_mmp_name;346 u32 fuses;347 u32 straps;348 u32 accel_capabilities_mask;349 u32 extended_dc_capabilities;350 u16 fw_capabilities;351 u32 clock_frequency;352 u32 instance_id;353 u16 accel_mask;354 u32 ae_mask;355 u32 admin_ae_mask;356 u16 tx_rings_mask;357 u16 ring_to_svc_map;358 u32 thd_to_arb_map[ICP_QAT_HW_AE_DELIMITER];359 u8 tx_rx_gap;360 u8 num_banks;361 u16 num_banks_per_vf;362 u8 num_rings_per_bank;363 u8 num_accel;364 u8 num_logical_accel;365 u8 num_engines;366 u32 num_hb_ctrs;367 u8 num_rps;368};369 370/* CSR write macro */371#define ADF_CSR_WR(csr_base, csr_offset, val) \372 __raw_writel(val, csr_base + csr_offset)373 374/* CSR read macro */375#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset)376 377#define ADF_CFG_NUM_SERVICES 4378#define ADF_SRV_TYPE_BIT_LEN 3379#define ADF_SRV_TYPE_MASK 0x7380#define ADF_AE_ADMIN_THREAD 7381#define ADF_NUM_THREADS_PER_AE 8382#define ADF_NUM_PKE_STRAND 2383#define ADF_AE_STRAND0_THREAD 8384#define ADF_AE_STRAND1_THREAD 9385 386#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev)387#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars)388#define GET_HW_DATA(accel_dev) (accel_dev->hw_device)389#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks)390#define GET_NUM_RINGS_PER_BANK(accel_dev) \391 GET_HW_DATA(accel_dev)->num_rings_per_bank392#define GET_SRV_TYPE(accel_dev, idx) \393 (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \394 & ADF_SRV_TYPE_MASK)395#define GET_ERR_MASK(accel_dev) (&GET_HW_DATA(accel_dev)->dev_err_mask)396#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines)397#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops)398#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops)399#define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops)400#define GET_VFMIG_OPS(accel_dev) (&(accel_dev)->hw_device->vfmig_ops)401#define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data402#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev403 404struct adf_admin_comms;405struct icp_qat_fw_loader_handle;406struct adf_fw_loader_data {407 struct icp_qat_fw_loader_handle *fw_loader;408 const struct firmware *uof_fw;409 const struct firmware *mmp_fw;410};411 412struct adf_accel_vf_info {413 struct adf_accel_dev *accel_dev;414 struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */415 struct mutex pfvf_mig_lock; /* protects PFVF state for migration */416 struct ratelimit_state vf2pf_ratelimit;417 u32 vf_nr;418 bool init;419 bool restarting;420 u8 vf_compat_ver;421 /*422 * Private area used for device migration.423 * Memory allocation and free is managed by migration driver.424 */425 void *mig_priv;426};427 428struct adf_dc_data {429 u8 *ovf_buff;430 size_t ovf_buff_sz;431 dma_addr_t ovf_buff_p;432};433 434struct adf_pm {435 struct dentry *debugfs_pm_status;436 bool present;437 int idle_irq_counters;438 int throttle_irq_counters;439 int fw_irq_counters;440 int host_ack_counter;441 int host_nack_counter;442 ssize_t (*print_pm_status)(struct adf_accel_dev *accel_dev,443 char __user *buf, size_t count, loff_t *pos);444};445 446struct adf_sysfs {447 int ring_num;448 struct rw_semaphore lock; /* protects access to the fields in this struct */449};450 451struct adf_accel_dev {452 struct adf_etr_data *transport;453 struct adf_hw_device_data *hw_device;454 struct adf_cfg_device_data *cfg;455 struct adf_fw_loader_data *fw_loader;456 struct adf_admin_comms *admin;457 struct adf_telemetry *telemetry;458 struct adf_dc_data *dc_data;459 struct adf_pm power_management;460 struct list_head crypto_list;461 struct list_head compression_list;462 unsigned long status;463 atomic_t ref_count;464 struct dentry *debugfs_dir;465 struct dentry *fw_cntr_dbgfile;466 struct dentry *cnv_dbgfile;467 struct list_head list;468 struct module *owner;469 struct adf_accel_pci accel_pci_dev;470 struct adf_timer *timer;471 struct adf_heartbeat *heartbeat;472 struct adf_rl *rate_limiting;473 struct adf_sysfs sysfs;474 union {475 struct {476 /* protects VF2PF interrupts access */477 spinlock_t vf2pf_ints_lock;478 /* vf_info is non-zero when SR-IOV is init'ed */479 struct adf_accel_vf_info *vf_info;480 } pf;481 struct {482 bool irq_enabled;483 char irq_name[ADF_MAX_MSIX_VECTOR_NAME];484 struct tasklet_struct pf2vf_bh_tasklet;485 struct mutex vf2pf_lock; /* protect CSR access */486 struct completion msg_received;487 struct pfvf_message response; /* temp field holding pf2vf response */488 u8 pf_compat_ver;489 } vf;490 };491 struct adf_error_counters ras_errors;492 struct mutex state_lock; /* protect state of the device */493 bool is_vf;494 bool autoreset_on_error;495 u32 accel_id;496};497#endif498