1443 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 HiSilicon Limited. */3#include <linux/acpi.h>4#include <linux/bitops.h>5#include <linux/debugfs.h>6#include <linux/init.h>7#include <linux/io.h>8#include <linux/kernel.h>9#include <linux/module.h>10#include <linux/pci.h>11#include <linux/pm_runtime.h>12#include <linux/seq_file.h>13#include <linux/topology.h>14#include <linux/uacce.h>15#include "zip.h"16 17#define PCI_DEVICE_ID_HUAWEI_ZIP_PF 0xa25018 19#define HZIP_QUEUE_NUM_V1 409620 21#define HZIP_CLOCK_GATE_CTRL 0x30100422#define HZIP_DECOMP_CHECK_ENABLE BIT(16)23#define HZIP_FSM_MAX_CNT 0x30100824 25#define HZIP_PORT_ARCA_CHE_0 0x30104026#define HZIP_PORT_ARCA_CHE_1 0x30104427#define HZIP_PORT_AWCA_CHE_0 0x30106028#define HZIP_PORT_AWCA_CHE_1 0x30106429#define HZIP_CACHE_ALL_EN 0xffffffff30 31#define HZIP_BD_RUSER_32_63 0x30111032#define HZIP_SGL_RUSER_32_63 0x30111c33#define HZIP_DATA_RUSER_32_63 0x30112834#define HZIP_DATA_WUSER_32_63 0x30113435#define HZIP_BD_WUSER_32_63 0x30114036 37#define HZIP_QM_IDEL_STATUS 0x3040e438 39#define HZIP_CORE_DFX_BASE 0x30100040#define HZIP_CORE_DFX_DECOMP_BASE 0x30400041#define HZIP_CORE_DFX_COMP_0 0x30200042#define HZIP_CORE_DFX_COMP_1 0x30300043#define HZIP_CORE_DFX_DECOMP_0 0x30400044#define HZIP_CORE_DFX_DECOMP_1 0x30500045#define HZIP_CORE_DFX_DECOMP_2 0x30600046#define HZIP_CORE_DFX_DECOMP_3 0x30700047#define HZIP_CORE_DFX_DECOMP_4 0x30800048#define HZIP_CORE_DFX_DECOMP_5 0x30900049#define HZIP_CORE_REGS_BASE_LEN 0xB050#define HZIP_CORE_REGS_DFX_LEN 0x2851#define HZIP_CORE_ADDR_INTRVL 0x100052 53#define HZIP_CORE_INT_SOURCE 0x3010A054#define HZIP_CORE_INT_MASK_REG 0x3010A455#define HZIP_CORE_INT_SET 0x3010A856#define HZIP_CORE_INT_STATUS 0x3010AC57#define HZIP_CORE_INT_STATUS_M_ECC BIT(1)58#define HZIP_CORE_SRAM_ECC_ERR_INFO 0x30114859#define HZIP_CORE_INT_RAS_CE_ENB 0x30116060#define HZIP_CORE_INT_RAS_NFE_ENB 0x30116461#define HZIP_CORE_INT_RAS_FE_ENB 0x30116862#define HZIP_CORE_INT_RAS_FE_ENB_MASK 0x063#define HZIP_OOO_SHUTDOWN_SEL 0x30120C64#define HZIP_SRAM_ECC_ERR_NUM_SHIFT 1665#define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 2466#define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0)67#define HZIP_SQE_SIZE 12868#define HZIP_PF_DEF_Q_NUM 6469#define HZIP_PF_DEF_Q_BASE 070#define HZIP_CTX_Q_NUM_DEF 271 72#define HZIP_SOFT_CTRL_CNT_CLR_CE 0x30100073#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0)74#define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C75#define HZIP_AXI_SHUTDOWN_ENABLE BIT(14)76#define HZIP_WR_PORT BIT(11)77 78#define HZIP_ALG_ZLIB_BIT GENMASK(1, 0)79#define HZIP_ALG_GZIP_BIT GENMASK(3, 2)80#define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4)81#define HZIP_ALG_LZ77_BIT GENMASK(7, 6)82 83#define HZIP_BUF_SIZE 2284#define HZIP_SQE_MASK_OFFSET 6485#define HZIP_SQE_MASK_LEN 4886 87#define HZIP_CNT_CLR_CE_EN BIT(0)88#define HZIP_RO_CNT_CLR_CE_EN BIT(2)89#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \90 HZIP_RO_CNT_CLR_CE_EN)91 92#define HZIP_PREFETCH_CFG 0x3011B093#define HZIP_SVA_TRANS 0x3011C494#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0)))95#define HZIP_SVA_PREFETCH_DISABLE BIT(26)96#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30))97#define HZIP_SHAPER_RATE_COMPRESS 75098#define HZIP_SHAPER_RATE_DECOMPRESS 14099#define HZIP_DELAY_1_US 1100#define HZIP_POLL_TIMEOUT_US 1000101 102/* clock gating */103#define HZIP_PEH_CFG_AUTO_GATE 0x3011A8104#define HZIP_PEH_CFG_AUTO_GATE_EN BIT(0)105#define HZIP_CORE_GATED_EN GENMASK(15, 8)106#define HZIP_CORE_GATED_OOO_EN BIT(29)107#define HZIP_CLOCK_GATED_EN (HZIP_CORE_GATED_EN | \108 HZIP_CORE_GATED_OOO_EN)109 110/* zip comp high performance */111#define HZIP_HIGH_PERF_OFFSET 0x301208112 113enum {114 HZIP_HIGH_COMP_RATE,115 HZIP_HIGH_COMP_PERF,116};117 118static const char hisi_zip_name[] = "hisi_zip";119static struct dentry *hzip_debugfs_root;120 121struct hisi_zip_hw_error {122 u32 int_msk;123 const char *msg;124};125 126struct zip_dfx_item {127 const char *name;128 u32 offset;129};130 131static const struct qm_dev_alg zip_dev_algs[] = { {132 .alg_msk = HZIP_ALG_ZLIB_BIT,133 .alg = "zlib\n",134 }, {135 .alg_msk = HZIP_ALG_GZIP_BIT,136 .alg = "gzip\n",137 }, {138 .alg_msk = HZIP_ALG_DEFLATE_BIT,139 .alg = "deflate\n",140 }, {141 .alg_msk = HZIP_ALG_LZ77_BIT,142 .alg = "lz77_zstd\n",143 },144};145 146static struct hisi_qm_list zip_devices = {147 .register_to_crypto = hisi_zip_register_to_crypto,148 .unregister_from_crypto = hisi_zip_unregister_from_crypto,149};150 151static struct zip_dfx_item zip_dfx_files[] = {152 {"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)},153 {"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)},154 {"send_busy_cnt", offsetof(struct hisi_zip_dfx, send_busy_cnt)},155 {"err_bd_cnt", offsetof(struct hisi_zip_dfx, err_bd_cnt)},156};157 158static const struct hisi_zip_hw_error zip_hw_error[] = {159 { .int_msk = BIT(0), .msg = "zip_ecc_1bitt_err" },160 { .int_msk = BIT(1), .msg = "zip_ecc_2bit_err" },161 { .int_msk = BIT(2), .msg = "zip_axi_rresp_err" },162 { .int_msk = BIT(3), .msg = "zip_axi_bresp_err" },163 { .int_msk = BIT(4), .msg = "zip_src_addr_parse_err" },164 { .int_msk = BIT(5), .msg = "zip_dst_addr_parse_err" },165 { .int_msk = BIT(6), .msg = "zip_pre_in_addr_err" },166 { .int_msk = BIT(7), .msg = "zip_pre_in_data_err" },167 { .int_msk = BIT(8), .msg = "zip_com_inf_err" },168 { .int_msk = BIT(9), .msg = "zip_enc_inf_err" },169 { .int_msk = BIT(10), .msg = "zip_pre_out_err" },170 { .int_msk = BIT(11), .msg = "zip_axi_poison_err" },171 { .int_msk = BIT(12), .msg = "zip_sva_err" },172 { /* sentinel */ }173};174 175enum ctrl_debug_file_index {176 HZIP_CLEAR_ENABLE,177 HZIP_DEBUG_FILE_NUM,178};179 180static const char * const ctrl_debug_file_name[] = {181 [HZIP_CLEAR_ENABLE] = "clear_enable",182};183 184struct ctrl_debug_file {185 enum ctrl_debug_file_index index;186 spinlock_t lock;187 struct hisi_zip_ctrl *ctrl;188};189 190/*191 * One ZIP controller has one PF and multiple VFs, some global configurations192 * which PF has need this structure.193 *194 * Just relevant for PF.195 */196struct hisi_zip_ctrl {197 struct hisi_zip *hisi_zip;198 struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];199};200 201enum zip_cap_type {202 ZIP_QM_NFE_MASK_CAP = 0x0,203 ZIP_QM_RESET_MASK_CAP,204 ZIP_QM_OOO_SHUTDOWN_MASK_CAP,205 ZIP_QM_CE_MASK_CAP,206 ZIP_NFE_MASK_CAP,207 ZIP_RESET_MASK_CAP,208 ZIP_OOO_SHUTDOWN_MASK_CAP,209 ZIP_CE_MASK_CAP,210 ZIP_CLUSTER_NUM_CAP,211 ZIP_CORE_TYPE_NUM_CAP,212 ZIP_CORE_NUM_CAP,213 ZIP_CLUSTER_COMP_NUM_CAP,214 ZIP_CLUSTER_DECOMP_NUM_CAP,215 ZIP_DECOMP_ENABLE_BITMAP,216 ZIP_COMP_ENABLE_BITMAP,217 ZIP_DRV_ALG_BITMAP,218 ZIP_DEV_ALG_BITMAP,219 ZIP_CORE1_ALG_BITMAP,220 ZIP_CORE2_ALG_BITMAP,221 ZIP_CORE3_ALG_BITMAP,222 ZIP_CORE4_ALG_BITMAP,223 ZIP_CORE5_ALG_BITMAP,224 ZIP_CAP_MAX225};226 227static struct hisi_qm_cap_info zip_basic_cap_info[] = {228 {ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77},229 {ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77},230 {ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77},231 {ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},232 {ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE},233 {ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE},234 {ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE},235 {ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1},236 {ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1},237 {ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2},238 {ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5},239 {ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2},240 {ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3},241 {ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C},242 {ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3},243 {ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x0, 0x30},244 {ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0x3F},245 {ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},246 {ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5},247 {ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},248 {ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},249 {ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A},250 {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0}251};252 253enum zip_pre_store_cap_idx {254 ZIP_CORE_NUM_CAP_IDX = 0x0,255 ZIP_CLUSTER_COMP_NUM_CAP_IDX,256 ZIP_CLUSTER_DECOMP_NUM_CAP_IDX,257 ZIP_DECOMP_ENABLE_BITMAP_IDX,258 ZIP_COMP_ENABLE_BITMAP_IDX,259 ZIP_DRV_ALG_BITMAP_IDX,260 ZIP_DEV_ALG_BITMAP_IDX,261};262 263static const u32 zip_pre_store_caps[] = {264 ZIP_CORE_NUM_CAP,265 ZIP_CLUSTER_COMP_NUM_CAP,266 ZIP_CLUSTER_DECOMP_NUM_CAP,267 ZIP_DECOMP_ENABLE_BITMAP,268 ZIP_COMP_ENABLE_BITMAP,269 ZIP_DRV_ALG_BITMAP,270 ZIP_DEV_ALG_BITMAP,271};272 273static const struct debugfs_reg32 hzip_dfx_regs[] = {274 {"HZIP_GET_BD_NUM ", 0x00},275 {"HZIP_GET_RIGHT_BD ", 0x04},276 {"HZIP_GET_ERROR_BD ", 0x08},277 {"HZIP_DONE_BD_NUM ", 0x0c},278 {"HZIP_WORK_CYCLE ", 0x10},279 {"HZIP_IDLE_CYCLE ", 0x18},280 {"HZIP_MAX_DELAY ", 0x20},281 {"HZIP_MIN_DELAY ", 0x24},282 {"HZIP_AVG_DELAY ", 0x28},283 {"HZIP_MEM_VISIBLE_DATA ", 0x30},284 {"HZIP_MEM_VISIBLE_ADDR ", 0x34},285 {"HZIP_CONSUMED_BYTE ", 0x38},286 {"HZIP_PRODUCED_BYTE ", 0x40},287 {"HZIP_COMP_INF ", 0x70},288 {"HZIP_PRE_OUT ", 0x78},289 {"HZIP_BD_RD ", 0x7c},290 {"HZIP_BD_WR ", 0x80},291 {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84},292 {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88},293 {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8c},294 {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94},295 {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9c},296};297 298static const struct debugfs_reg32 hzip_com_dfx_regs[] = {299 {"HZIP_CLOCK_GATE_CTRL ", 0x301004},300 {"HZIP_CORE_INT_RAS_CE_ENB ", 0x301160},301 {"HZIP_CORE_INT_RAS_NFE_ENB ", 0x301164},302 {"HZIP_CORE_INT_RAS_FE_ENB ", 0x301168},303 {"HZIP_UNCOM_ERR_RAS_CTRL ", 0x30116C},304};305 306static const struct debugfs_reg32 hzip_dump_dfx_regs[] = {307 {"HZIP_GET_BD_NUM ", 0x00},308 {"HZIP_GET_RIGHT_BD ", 0x04},309 {"HZIP_GET_ERROR_BD ", 0x08},310 {"HZIP_DONE_BD_NUM ", 0x0c},311 {"HZIP_MAX_DELAY ", 0x20},312};313 314/* define the ZIP's dfx regs region and region length */315static struct dfx_diff_registers hzip_diff_regs[] = {316 {317 .reg_offset = HZIP_CORE_DFX_BASE,318 .reg_len = HZIP_CORE_REGS_BASE_LEN,319 }, {320 .reg_offset = HZIP_CORE_DFX_COMP_0,321 .reg_len = HZIP_CORE_REGS_DFX_LEN,322 }, {323 .reg_offset = HZIP_CORE_DFX_COMP_1,324 .reg_len = HZIP_CORE_REGS_DFX_LEN,325 }, {326 .reg_offset = HZIP_CORE_DFX_DECOMP_0,327 .reg_len = HZIP_CORE_REGS_DFX_LEN,328 }, {329 .reg_offset = HZIP_CORE_DFX_DECOMP_1,330 .reg_len = HZIP_CORE_REGS_DFX_LEN,331 }, {332 .reg_offset = HZIP_CORE_DFX_DECOMP_2,333 .reg_len = HZIP_CORE_REGS_DFX_LEN,334 }, {335 .reg_offset = HZIP_CORE_DFX_DECOMP_3,336 .reg_len = HZIP_CORE_REGS_DFX_LEN,337 }, {338 .reg_offset = HZIP_CORE_DFX_DECOMP_4,339 .reg_len = HZIP_CORE_REGS_DFX_LEN,340 }, {341 .reg_offset = HZIP_CORE_DFX_DECOMP_5,342 .reg_len = HZIP_CORE_REGS_DFX_LEN,343 },344};345 346static int hzip_diff_regs_show(struct seq_file *s, void *unused)347{348 struct hisi_qm *qm = s->private;349 350 hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,351 ARRAY_SIZE(hzip_diff_regs));352 353 return 0;354}355DEFINE_SHOW_ATTRIBUTE(hzip_diff_regs);356 357static int perf_mode_set(const char *val, const struct kernel_param *kp)358{359 int ret;360 u32 n;361 362 if (!val)363 return -EINVAL;364 365 ret = kstrtou32(val, 10, &n);366 if (ret != 0 || (n != HZIP_HIGH_COMP_PERF &&367 n != HZIP_HIGH_COMP_RATE))368 return -EINVAL;369 370 return param_set_int(val, kp);371}372 373static const struct kernel_param_ops zip_com_perf_ops = {374 .set = perf_mode_set,375 .get = param_get_int,376};377 378/*379 * perf_mode = 0 means enable high compression rate mode,380 * perf_mode = 1 means enable high compression performance mode.381 * These two modes only apply to the compression direction.382 */383static u32 perf_mode = HZIP_HIGH_COMP_RATE;384module_param_cb(perf_mode, &zip_com_perf_ops, &perf_mode, 0444);385MODULE_PARM_DESC(perf_mode, "ZIP high perf mode 0(default), 1(enable)");386 387static const struct kernel_param_ops zip_uacce_mode_ops = {388 .set = uacce_mode_set,389 .get = param_get_int,390};391 392/*393 * uacce_mode = 0 means zip only register to crypto,394 * uacce_mode = 1 means zip both register to crypto and uacce.395 */396static u32 uacce_mode = UACCE_MODE_NOUACCE;397module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444);398MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);399 400static bool pf_q_num_flag;401static int pf_q_num_set(const char *val, const struct kernel_param *kp)402{403 pf_q_num_flag = true;404 405 return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF);406}407 408static const struct kernel_param_ops pf_q_num_ops = {409 .set = pf_q_num_set,410 .get = param_get_int,411};412 413static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;414module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);415MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");416 417static const struct kernel_param_ops vfs_num_ops = {418 .set = vfs_num_set,419 .get = param_get_int,420};421 422static u32 vfs_num;423module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);424MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");425 426static const struct pci_device_id hisi_zip_dev_ids[] = {427 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_PF) },428 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },429 { 0, }430};431MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids);432 433int zip_create_qps(struct hisi_qp **qps, int qp_num, int node)434{435 if (node == NUMA_NO_NODE)436 node = cpu_to_node(raw_smp_processor_id());437 438 return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps);439}440 441bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg)442{443 u32 cap_val;444 445 cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_IDX].cap_val;446 if ((alg & cap_val) == alg)447 return true;448 449 return false;450}451 452static int hisi_zip_set_high_perf(struct hisi_qm *qm)453{454 u32 val;455 int ret;456 457 val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET);458 if (perf_mode == HZIP_HIGH_COMP_PERF)459 val |= HZIP_HIGH_COMP_PERF;460 else461 val &= ~HZIP_HIGH_COMP_PERF;462 463 /* Set perf mode */464 writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET);465 ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET,466 val, val == perf_mode, HZIP_DELAY_1_US,467 HZIP_POLL_TIMEOUT_US);468 if (ret)469 pci_err(qm->pdev, "failed to set perf mode\n");470 471 return ret;472}473 474static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm)475{476 u32 val;477 int ret;478 479 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))480 return;481 482 /* Enable prefetch */483 val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);484 val &= HZIP_PREFETCH_ENABLE;485 writel(val, qm->io_base + HZIP_PREFETCH_CFG);486 487 ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG,488 val, !(val & HZIP_SVA_PREFETCH_DISABLE),489 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);490 if (ret)491 pci_err(qm->pdev, "failed to open sva prefetch\n");492}493 494static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm)495{496 u32 val;497 int ret;498 499 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))500 return;501 502 val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG);503 val |= HZIP_SVA_PREFETCH_DISABLE;504 writel(val, qm->io_base + HZIP_PREFETCH_CFG);505 506 ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS,507 val, !(val & HZIP_SVA_DISABLE_READY),508 HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US);509 if (ret)510 pci_err(qm->pdev, "failed to close sva prefetch\n");511}512 513static void hisi_zip_enable_clock_gate(struct hisi_qm *qm)514{515 u32 val;516 517 if (qm->ver < QM_HW_V3)518 return;519 520 val = readl(qm->io_base + HZIP_CLOCK_GATE_CTRL);521 val |= HZIP_CLOCK_GATED_EN;522 writel(val, qm->io_base + HZIP_CLOCK_GATE_CTRL);523 524 val = readl(qm->io_base + HZIP_PEH_CFG_AUTO_GATE);525 val |= HZIP_PEH_CFG_AUTO_GATE_EN;526 writel(val, qm->io_base + HZIP_PEH_CFG_AUTO_GATE);527}528 529static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)530{531 void __iomem *base = qm->io_base;532 u32 dcomp_bm, comp_bm;533 534 /* qm user domain */535 writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1);536 writel(ARUSER_M_CFG_ENABLE, base + QM_ARUSER_M_CFG_ENABLE);537 writel(AXUSER_BASE, base + QM_AWUSER_M_CFG_1);538 writel(AWUSER_M_CFG_ENABLE, base + QM_AWUSER_M_CFG_ENABLE);539 writel(WUSER_M_CFG_ENABLE, base + QM_WUSER_M_CFG_ENABLE);540 541 /* qm cache */542 writel(AXI_M_CFG, base + QM_AXI_M_CFG);543 writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE);544 545 /* disable FLR triggered by BME(bus master enable) */546 writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG);547 writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE);548 549 /* cache */550 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);551 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);552 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);553 writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);554 555 /* user domain configurations */556 writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);557 writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);558 559 if (qm->use_sva && qm->ver == QM_HW_V2) {560 writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);561 writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);562 writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);563 } else {564 writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);565 writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);566 writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);567 }568 569 /* let's open all compression/decompression cores */570 dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val;571 comp_bm = qm->cap_tables.dev_cap_table[ZIP_COMP_ENABLE_BITMAP_IDX].cap_val;572 writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL);573 574 /* enable sqc,cqc writeback */575 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |576 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |577 FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);578 579 hisi_zip_enable_clock_gate(qm);580 581 return 0;582}583 584static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable)585{586 u32 val1, val2;587 588 val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);589 if (enable) {590 val1 |= HZIP_AXI_SHUTDOWN_ENABLE;591 val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info,592 ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);593 } else {594 val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE;595 val2 = 0x0;596 }597 598 if (qm->ver > QM_HW_V2)599 writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL);600 601 writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);602}603 604static void hisi_zip_hw_error_enable(struct hisi_qm *qm)605{606 u32 nfe, ce;607 608 if (qm->ver == QM_HW_V1) {609 writel(HZIP_CORE_INT_MASK_ALL,610 qm->io_base + HZIP_CORE_INT_MASK_REG);611 dev_info(&qm->pdev->dev, "Does not support hw error handle\n");612 return;613 }614 615 nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);616 ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);617 618 /* clear ZIP hw error source if having */619 writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE);620 621 /* configure error type */622 writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB);623 writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB);624 writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);625 626 hisi_zip_master_ooo_ctrl(qm, true);627 628 /* enable ZIP hw error interrupts */629 writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG);630}631 632static void hisi_zip_hw_error_disable(struct hisi_qm *qm)633{634 u32 nfe, ce;635 636 /* disable ZIP hw error interrupts */637 nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);638 ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver);639 writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG);640 641 hisi_zip_master_ooo_ctrl(qm, false);642}643 644static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)645{646 struct hisi_zip *hisi_zip = file->ctrl->hisi_zip;647 648 return &hisi_zip->qm;649}650 651static u32 clear_enable_read(struct hisi_qm *qm)652{653 return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &654 HZIP_SOFT_CTRL_CNT_CLR_CE_BIT;655}656 657static int clear_enable_write(struct hisi_qm *qm, u32 val)658{659 u32 tmp;660 661 if (val != 1 && val != 0)662 return -EINVAL;663 664 tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &665 ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val;666 writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);667 668 return 0;669}670 671static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf,672 size_t count, loff_t *pos)673{674 struct ctrl_debug_file *file = filp->private_data;675 struct hisi_qm *qm = file_to_qm(file);676 char tbuf[HZIP_BUF_SIZE];677 u32 val;678 int ret;679 680 ret = hisi_qm_get_dfx_access(qm);681 if (ret)682 return ret;683 684 spin_lock_irq(&file->lock);685 switch (file->index) {686 case HZIP_CLEAR_ENABLE:687 val = clear_enable_read(qm);688 break;689 default:690 goto err_input;691 }692 spin_unlock_irq(&file->lock);693 694 hisi_qm_put_dfx_access(qm);695 ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val);696 return simple_read_from_buffer(buf, count, pos, tbuf, ret);697 698err_input:699 spin_unlock_irq(&file->lock);700 hisi_qm_put_dfx_access(qm);701 return -EINVAL;702}703 704static ssize_t hisi_zip_ctrl_debug_write(struct file *filp,705 const char __user *buf,706 size_t count, loff_t *pos)707{708 struct ctrl_debug_file *file = filp->private_data;709 struct hisi_qm *qm = file_to_qm(file);710 char tbuf[HZIP_BUF_SIZE];711 unsigned long val;712 int len, ret;713 714 if (*pos != 0)715 return 0;716 717 if (count >= HZIP_BUF_SIZE)718 return -ENOSPC;719 720 len = simple_write_to_buffer(tbuf, HZIP_BUF_SIZE - 1, pos, buf, count);721 if (len < 0)722 return len;723 724 tbuf[len] = '\0';725 ret = kstrtoul(tbuf, 0, &val);726 if (ret)727 return ret;728 729 ret = hisi_qm_get_dfx_access(qm);730 if (ret)731 return ret;732 733 spin_lock_irq(&file->lock);734 switch (file->index) {735 case HZIP_CLEAR_ENABLE:736 ret = clear_enable_write(qm, val);737 if (ret)738 goto err_input;739 break;740 default:741 ret = -EINVAL;742 goto err_input;743 }744 745 ret = count;746 747err_input:748 spin_unlock_irq(&file->lock);749 hisi_qm_put_dfx_access(qm);750 return ret;751}752 753static const struct file_operations ctrl_debug_fops = {754 .owner = THIS_MODULE,755 .open = simple_open,756 .read = hisi_zip_ctrl_debug_read,757 .write = hisi_zip_ctrl_debug_write,758};759 760static int zip_debugfs_atomic64_set(void *data, u64 val)761{762 if (val)763 return -EINVAL;764 765 atomic64_set((atomic64_t *)data, 0);766 767 return 0;768}769 770static int zip_debugfs_atomic64_get(void *data, u64 *val)771{772 *val = atomic64_read((atomic64_t *)data);773 774 return 0;775}776 777DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get,778 zip_debugfs_atomic64_set, "%llu\n");779 780static int hisi_zip_regs_show(struct seq_file *s, void *unused)781{782 hisi_qm_regs_dump(s, s->private);783 784 return 0;785}786 787DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs);788 789static void __iomem *get_zip_core_addr(struct hisi_qm *qm, int core_num)790{791 u32 zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;792 793 if (core_num < zip_comp_core_num)794 return qm->io_base + HZIP_CORE_DFX_BASE +795 (core_num + 1) * HZIP_CORE_ADDR_INTRVL;796 797 return qm->io_base + HZIP_CORE_DFX_DECOMP_BASE +798 (core_num - zip_comp_core_num) * HZIP_CORE_ADDR_INTRVL;799}800 801static int hisi_zip_core_debug_init(struct hisi_qm *qm)802{803 u32 zip_core_num, zip_comp_core_num;804 struct device *dev = &qm->pdev->dev;805 struct debugfs_regset32 *regset;806 struct dentry *tmp_d;807 char buf[HZIP_BUF_SIZE];808 int i;809 810 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;811 zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;812 813 for (i = 0; i < zip_core_num; i++) {814 if (i < zip_comp_core_num)815 scnprintf(buf, sizeof(buf), "comp_core%d", i);816 else817 scnprintf(buf, sizeof(buf), "decomp_core%d",818 i - zip_comp_core_num);819 820 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);821 if (!regset)822 return -ENOENT;823 824 regset->regs = hzip_dfx_regs;825 regset->nregs = ARRAY_SIZE(hzip_dfx_regs);826 regset->base = get_zip_core_addr(qm, i);827 regset->dev = dev;828 829 tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);830 debugfs_create_file("regs", 0444, tmp_d, regset,831 &hisi_zip_regs_fops);832 }833 834 return 0;835}836 837static void hisi_zip_dfx_debug_init(struct hisi_qm *qm)838{839 struct dfx_diff_registers *hzip_regs = qm->debug.acc_diff_regs;840 struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);841 struct hisi_zip_dfx *dfx = &zip->dfx;842 struct dentry *tmp_dir;843 void *data;844 int i;845 846 tmp_dir = debugfs_create_dir("zip_dfx", qm->debug.debug_root);847 for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) {848 data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset);849 debugfs_create_file(zip_dfx_files[i].name,850 0644, tmp_dir, data,851 &zip_atomic64_ops);852 }853 854 if (qm->fun_type == QM_HW_PF && hzip_regs)855 debugfs_create_file("diff_regs", 0444, tmp_dir,856 qm, &hzip_diff_regs_fops);857}858 859static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm)860{861 struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);862 int i;863 864 for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) {865 spin_lock_init(&zip->ctrl->files[i].lock);866 zip->ctrl->files[i].ctrl = zip->ctrl;867 zip->ctrl->files[i].index = i;868 869 debugfs_create_file(ctrl_debug_file_name[i], 0600,870 qm->debug.debug_root,871 zip->ctrl->files + i,872 &ctrl_debug_fops);873 }874 875 return hisi_zip_core_debug_init(qm);876}877 878static int hisi_zip_debugfs_init(struct hisi_qm *qm)879{880 struct device *dev = &qm->pdev->dev;881 int ret;882 883 ret = hisi_qm_regs_debugfs_init(qm, hzip_diff_regs, ARRAY_SIZE(hzip_diff_regs));884 if (ret) {885 dev_warn(dev, "Failed to init ZIP diff regs!\n");886 return ret;887 }888 889 qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET;890 qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN;891 qm->debug.debug_root = debugfs_create_dir(dev_name(dev),892 hzip_debugfs_root);893 894 hisi_qm_debug_init(qm);895 896 if (qm->fun_type == QM_HW_PF) {897 ret = hisi_zip_ctrl_debug_init(qm);898 if (ret)899 goto debugfs_remove;900 }901 902 hisi_zip_dfx_debug_init(qm);903 904 return 0;905 906debugfs_remove:907 debugfs_remove_recursive(qm->debug.debug_root);908 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));909 return ret;910}911 912/* hisi_zip_debug_regs_clear() - clear the zip debug regs */913static void hisi_zip_debug_regs_clear(struct hisi_qm *qm)914{915 u32 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;916 int i, j;917 918 /* enable register read_clear bit */919 writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);920 for (i = 0; i < zip_core_num; i++)921 for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++)922 readl(get_zip_core_addr(qm, i) +923 hzip_dfx_regs[j].offset);924 925 /* disable register read_clear bit */926 writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);927 928 hisi_qm_debug_regs_clear(qm);929}930 931static void hisi_zip_debugfs_exit(struct hisi_qm *qm)932{933 debugfs_remove_recursive(qm->debug.debug_root);934 935 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hzip_diff_regs));936 937 if (qm->fun_type == QM_HW_PF) {938 hisi_zip_debug_regs_clear(qm);939 qm->debug.curr_qm_qp_num = 0;940 }941}942 943static int hisi_zip_show_last_regs_init(struct hisi_qm *qm)944{945 int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs);946 int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);947 struct qm_debug *debug = &qm->debug;948 void __iomem *io_base;949 u32 zip_core_num;950 int i, j, idx;951 952 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;953 954 debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num,955 sizeof(unsigned int), GFP_KERNEL);956 if (!debug->last_words)957 return -ENOMEM;958 959 for (i = 0; i < com_dfx_regs_num; i++) {960 io_base = qm->io_base + hzip_com_dfx_regs[i].offset;961 debug->last_words[i] = readl_relaxed(io_base);962 }963 964 for (i = 0; i < zip_core_num; i++) {965 io_base = get_zip_core_addr(qm, i);966 for (j = 0; j < core_dfx_regs_num; j++) {967 idx = com_dfx_regs_num + i * core_dfx_regs_num + j;968 debug->last_words[idx] = readl_relaxed(969 io_base + hzip_dump_dfx_regs[j].offset);970 }971 }972 973 return 0;974}975 976static void hisi_zip_show_last_regs_uninit(struct hisi_qm *qm)977{978 struct qm_debug *debug = &qm->debug;979 980 if (qm->fun_type == QM_HW_VF || !debug->last_words)981 return;982 983 kfree(debug->last_words);984 debug->last_words = NULL;985}986 987static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm)988{989 int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs);990 int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs);991 u32 zip_core_num, zip_comp_core_num;992 struct qm_debug *debug = &qm->debug;993 char buf[HZIP_BUF_SIZE];994 void __iomem *base;995 int i, j, idx;996 u32 val;997 998 if (qm->fun_type == QM_HW_VF || !debug->last_words)999 return;1000 1001 for (i = 0; i < com_dfx_regs_num; i++) {1002 val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset);1003 if (debug->last_words[i] != val)1004 pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n",1005 hzip_com_dfx_regs[i].name, debug->last_words[i], val);1006 }1007 1008 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val;1009 zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val;1010 1011 for (i = 0; i < zip_core_num; i++) {1012 if (i < zip_comp_core_num)1013 scnprintf(buf, sizeof(buf), "Comp_core-%d", i);1014 else1015 scnprintf(buf, sizeof(buf), "Decomp_core-%d",1016 i - zip_comp_core_num);1017 base = get_zip_core_addr(qm, i);1018 1019 pci_info(qm->pdev, "==>%s:\n", buf);1020 /* dump last word for dfx regs during control resetting */1021 for (j = 0; j < core_dfx_regs_num; j++) {1022 idx = com_dfx_regs_num + i * core_dfx_regs_num + j;1023 val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset);1024 if (debug->last_words[idx] != val)1025 pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n",1026 hzip_dump_dfx_regs[j].name,1027 debug->last_words[idx], val);1028 }1029 }1030}1031 1032static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts)1033{1034 const struct hisi_zip_hw_error *err = zip_hw_error;1035 struct device *dev = &qm->pdev->dev;1036 u32 err_val;1037 1038 while (err->msg) {1039 if (err->int_msk & err_sts) {1040 dev_err(dev, "%s [error status=0x%x] found\n",1041 err->msg, err->int_msk);1042 1043 if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) {1044 err_val = readl(qm->io_base +1045 HZIP_CORE_SRAM_ECC_ERR_INFO);1046 dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n",1047 ((err_val >>1048 HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF));1049 }1050 }1051 err++;1052 }1053}1054 1055static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm)1056{1057 return readl(qm->io_base + HZIP_CORE_INT_STATUS);1058}1059 1060static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)1061{1062 u32 nfe;1063 1064 writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE);1065 nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver);1066 writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);1067}1068 1069static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm)1070{1071 u32 val;1072 1073 val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);1074 1075 writel(val & ~HZIP_AXI_SHUTDOWN_ENABLE,1076 qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);1077 1078 writel(val | HZIP_AXI_SHUTDOWN_ENABLE,1079 qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL);1080}1081 1082static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm)1083{1084 u32 nfe_enb;1085 1086 /* Disable ECC Mbit error report. */1087 nfe_enb = readl(qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);1088 writel(nfe_enb & ~HZIP_CORE_INT_STATUS_M_ECC,1089 qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);1090 1091 /* Inject zip ECC Mbit error to block master ooo. */1092 writel(HZIP_CORE_INT_STATUS_M_ECC,1093 qm->io_base + HZIP_CORE_INT_SET);1094}1095 1096static void hisi_zip_err_info_init(struct hisi_qm *qm)1097{1098 struct hisi_qm_err_info *err_info = &qm->err_info;1099 1100 err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK;1101 err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver);1102 err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info,1103 ZIP_QM_NFE_MASK_CAP, qm->cap_ver);1104 err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC;1105 err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,1106 ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);1107 err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,1108 ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);1109 err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,1110 ZIP_QM_RESET_MASK_CAP, qm->cap_ver);1111 err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info,1112 ZIP_RESET_MASK_CAP, qm->cap_ver);1113 err_info->msi_wr_port = HZIP_WR_PORT;1114 err_info->acpi_rst = "ZRST";1115}1116 1117static const struct hisi_qm_err_ini hisi_zip_err_ini = {1118 .hw_init = hisi_zip_set_user_domain_and_cache,1119 .hw_err_enable = hisi_zip_hw_error_enable,1120 .hw_err_disable = hisi_zip_hw_error_disable,1121 .get_dev_hw_err_status = hisi_zip_get_hw_err_status,1122 .clear_dev_hw_err_status = hisi_zip_clear_hw_err_status,1123 .log_dev_hw_err = hisi_zip_log_hw_error,1124 .open_axi_master_ooo = hisi_zip_open_axi_master_ooo,1125 .close_axi_master_ooo = hisi_zip_close_axi_master_ooo,1126 .open_sva_prefetch = hisi_zip_open_sva_prefetch,1127 .close_sva_prefetch = hisi_zip_close_sva_prefetch,1128 .show_last_dfx_regs = hisi_zip_show_last_dfx_regs,1129 .err_info_init = hisi_zip_err_info_init,1130};1131 1132static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)1133{1134 struct hisi_qm *qm = &hisi_zip->qm;1135 struct hisi_zip_ctrl *ctrl;1136 int ret;1137 1138 ctrl = devm_kzalloc(&qm->pdev->dev, sizeof(*ctrl), GFP_KERNEL);1139 if (!ctrl)1140 return -ENOMEM;1141 1142 hisi_zip->ctrl = ctrl;1143 ctrl->hisi_zip = hisi_zip;1144 1145 ret = hisi_zip_set_user_domain_and_cache(qm);1146 if (ret)1147 return ret;1148 1149 ret = hisi_zip_set_high_perf(qm);1150 if (ret)1151 return ret;1152 1153 hisi_zip_open_sva_prefetch(qm);1154 hisi_qm_dev_err_init(qm);1155 hisi_zip_debug_regs_clear(qm);1156 1157 ret = hisi_zip_show_last_regs_init(qm);1158 if (ret)1159 pci_err(qm->pdev, "Failed to init last word regs!\n");1160 1161 return ret;1162}1163 1164static int zip_pre_store_cap_reg(struct hisi_qm *qm)1165{1166 struct hisi_qm_cap_record *zip_cap;1167 struct pci_dev *pdev = qm->pdev;1168 size_t i, size;1169 1170 size = ARRAY_SIZE(zip_pre_store_caps);1171 zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL);1172 if (!zip_cap)1173 return -ENOMEM;1174 1175 for (i = 0; i < size; i++) {1176 zip_cap[i].type = zip_pre_store_caps[i];1177 zip_cap[i].cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info,1178 zip_pre_store_caps[i], qm->cap_ver);1179 }1180 1181 qm->cap_tables.dev_cap_table = zip_cap;1182 1183 return 0;1184}1185 1186static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)1187{1188 u64 alg_msk;1189 int ret;1190 1191 qm->pdev = pdev;1192 qm->ver = pdev->revision;1193 qm->mode = uacce_mode;1194 qm->sqe_size = HZIP_SQE_SIZE;1195 qm->dev_name = hisi_zip_name;1196 1197 qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_ZIP_PF) ?1198 QM_HW_PF : QM_HW_VF;1199 if (qm->fun_type == QM_HW_PF) {1200 qm->qp_base = HZIP_PF_DEF_Q_BASE;1201 qm->qp_num = pf_q_num;1202 qm->debug.curr_qm_qp_num = pf_q_num;1203 qm->qm_list = &zip_devices;1204 qm->err_ini = &hisi_zip_err_ini;1205 if (pf_q_num_flag)1206 set_bit(QM_MODULE_PARAM, &qm->misc_ctl);1207 } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {1208 /*1209 * have no way to get qm configure in VM in v1 hardware,1210 * so currently force PF to uses HZIP_PF_DEF_Q_NUM, and force1211 * to trigger only one VF in v1 hardware.1212 *1213 * v2 hardware has no such problem.1214 */1215 qm->qp_base = HZIP_PF_DEF_Q_NUM;1216 qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM;1217 }1218 1219 ret = hisi_qm_init(qm);1220 if (ret) {1221 pci_err(qm->pdev, "Failed to init zip qm configures!\n");1222 return ret;1223 }1224 1225 /* Fetch and save the value of capability registers */1226 ret = zip_pre_store_cap_reg(qm);1227 if (ret) {1228 pci_err(qm->pdev, "Failed to pre-store capability registers!\n");1229 hisi_qm_uninit(qm);1230 return ret;1231 }1232 1233 alg_msk = qm->cap_tables.dev_cap_table[ZIP_DEV_ALG_BITMAP_IDX].cap_val;1234 ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs));1235 if (ret) {1236 pci_err(qm->pdev, "Failed to set zip algs!\n");1237 hisi_qm_uninit(qm);1238 }1239 1240 return ret;1241}1242 1243static void hisi_zip_qm_uninit(struct hisi_qm *qm)1244{1245 hisi_qm_uninit(qm);1246}1247 1248static int hisi_zip_probe_init(struct hisi_zip *hisi_zip)1249{1250 u32 type_rate = HZIP_SHAPER_RATE_COMPRESS;1251 struct hisi_qm *qm = &hisi_zip->qm;1252 int ret;1253 1254 if (qm->fun_type == QM_HW_PF) {1255 ret = hisi_zip_pf_probe_init(hisi_zip);1256 if (ret)1257 return ret;1258 /* enable shaper type 0 */1259 if (qm->ver >= QM_HW_V3) {1260 type_rate |= QM_SHAPER_ENABLE;1261 1262 /* ZIP need to enable shaper type 1 */1263 type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET;1264 qm->type_rate = type_rate;1265 }1266 }1267 1268 return 0;1269}1270 1271static void hisi_zip_probe_uninit(struct hisi_qm *qm)1272{1273 if (qm->fun_type == QM_HW_VF)1274 return;1275 1276 hisi_zip_show_last_regs_uninit(qm);1277 hisi_zip_close_sva_prefetch(qm);1278 hisi_qm_dev_err_uninit(qm);1279}1280 1281static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)1282{1283 struct hisi_zip *hisi_zip;1284 struct hisi_qm *qm;1285 int ret;1286 1287 hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);1288 if (!hisi_zip)1289 return -ENOMEM;1290 1291 qm = &hisi_zip->qm;1292 1293 ret = hisi_zip_qm_init(qm, pdev);1294 if (ret) {1295 pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret);1296 return ret;1297 }1298 1299 ret = hisi_zip_probe_init(hisi_zip);1300 if (ret) {1301 pci_err(pdev, "Failed to probe (%d)!\n", ret);1302 goto err_qm_uninit;1303 }1304 1305 ret = hisi_qm_start(qm);1306 if (ret)1307 goto err_probe_uninit;1308 1309 ret = hisi_zip_debugfs_init(qm);1310 if (ret)1311 pci_err(pdev, "failed to init debugfs (%d)!\n", ret);1312 1313 hisi_qm_add_list(qm, &zip_devices);1314 ret = hisi_qm_alg_register(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);1315 if (ret < 0) {1316 pci_err(pdev, "failed to register driver to crypto!\n");1317 goto err_qm_del_list;1318 }1319 1320 if (qm->uacce) {1321 ret = uacce_register(qm->uacce);1322 if (ret) {1323 pci_err(pdev, "failed to register uacce (%d)!\n", ret);1324 goto err_qm_alg_unregister;1325 }1326 }1327 1328 if (qm->fun_type == QM_HW_PF && vfs_num > 0) {1329 ret = hisi_qm_sriov_enable(pdev, vfs_num);1330 if (ret < 0)1331 goto err_qm_alg_unregister;1332 }1333 1334 hisi_qm_pm_init(qm);1335 1336 return 0;1337 1338err_qm_alg_unregister:1339 hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);1340 1341err_qm_del_list:1342 hisi_qm_del_list(qm, &zip_devices);1343 hisi_zip_debugfs_exit(qm);1344 hisi_qm_stop(qm, QM_NORMAL);1345 1346err_probe_uninit:1347 hisi_zip_probe_uninit(qm);1348 1349err_qm_uninit:1350 hisi_zip_qm_uninit(qm);1351 1352 return ret;1353}1354 1355static void hisi_zip_remove(struct pci_dev *pdev)1356{1357 struct hisi_qm *qm = pci_get_drvdata(pdev);1358 1359 hisi_qm_pm_uninit(qm);1360 hisi_qm_wait_task_finish(qm, &zip_devices);1361 hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF);1362 hisi_qm_del_list(qm, &zip_devices);1363 1364 if (qm->fun_type == QM_HW_PF && qm->vfs_num)1365 hisi_qm_sriov_disable(pdev, true);1366 1367 hisi_zip_debugfs_exit(qm);1368 hisi_qm_stop(qm, QM_NORMAL);1369 hisi_zip_probe_uninit(qm);1370 hisi_zip_qm_uninit(qm);1371}1372 1373static const struct dev_pm_ops hisi_zip_pm_ops = {1374 SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)1375};1376 1377static const struct pci_error_handlers hisi_zip_err_handler = {1378 .error_detected = hisi_qm_dev_err_detected,1379 .slot_reset = hisi_qm_dev_slot_reset,1380 .reset_prepare = hisi_qm_reset_prepare,1381 .reset_done = hisi_qm_reset_done,1382};1383 1384static struct pci_driver hisi_zip_pci_driver = {1385 .name = "hisi_zip",1386 .id_table = hisi_zip_dev_ids,1387 .probe = hisi_zip_probe,1388 .remove = hisi_zip_remove,1389 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?1390 hisi_qm_sriov_configure : NULL,1391 .err_handler = &hisi_zip_err_handler,1392 .shutdown = hisi_qm_dev_shutdown,1393 .driver.pm = &hisi_zip_pm_ops,1394};1395 1396struct pci_driver *hisi_zip_get_pf_driver(void)1397{1398 return &hisi_zip_pci_driver;1399}1400EXPORT_SYMBOL_GPL(hisi_zip_get_pf_driver);1401 1402static void hisi_zip_register_debugfs(void)1403{1404 if (!debugfs_initialized())1405 return;1406 1407 hzip_debugfs_root = debugfs_create_dir("hisi_zip", NULL);1408}1409 1410static void hisi_zip_unregister_debugfs(void)1411{1412 debugfs_remove_recursive(hzip_debugfs_root);1413}1414 1415static int __init hisi_zip_init(void)1416{1417 int ret;1418 1419 hisi_qm_init_list(&zip_devices);1420 hisi_zip_register_debugfs();1421 1422 ret = pci_register_driver(&hisi_zip_pci_driver);1423 if (ret < 0) {1424 hisi_zip_unregister_debugfs();1425 pr_err("Failed to register pci driver.\n");1426 }1427 1428 return ret;1429}1430 1431static void __exit hisi_zip_exit(void)1432{1433 pci_unregister_driver(&hisi_zip_pci_driver);1434 hisi_zip_unregister_debugfs();1435}1436 1437module_init(hisi_zip_init);1438module_exit(hisi_zip_exit);1439 1440MODULE_LICENSE("GPL v2");1441MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");1442MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator");1443