1381 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 HiSilicon Limited. */3 4#include <linux/acpi.h>5#include <linux/bitops.h>6#include <linux/debugfs.h>7#include <linux/init.h>8#include <linux/io.h>9#include <linux/iommu.h>10#include <linux/kernel.h>11#include <linux/module.h>12#include <linux/pci.h>13#include <linux/pm_runtime.h>14#include <linux/seq_file.h>15#include <linux/topology.h>16#include <linux/uacce.h>17 18#include "sec.h"19 20#define SEC_VF_NUM 6321#define SEC_QUEUE_NUM_V1 409622#define PCI_DEVICE_ID_HUAWEI_SEC_PF 0xa25523 24#define SEC_BD_ERR_CHK_EN0 0xEFFFFFFF25#define SEC_BD_ERR_CHK_EN1 0x7ffff7fd26#define SEC_BD_ERR_CHK_EN3 0xffffbfff27 28#define SEC_SQE_SIZE 12829#define SEC_PF_DEF_Q_NUM 25630#define SEC_PF_DEF_Q_BASE 031#define SEC_CTX_Q_NUM_DEF 232#define SEC_CTX_Q_NUM_MAX 3233 34#define SEC_CTRL_CNT_CLR_CE 0x30112035#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0)36#define SEC_CORE_INT_SOURCE 0x30101037#define SEC_CORE_INT_MASK 0x30100038#define SEC_CORE_INT_STATUS 0x30100839#define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C1440#define SEC_ECC_NUM 1641#define SEC_ECC_MASH 0xFF42#define SEC_CORE_INT_DISABLE 0x043 44#define SEC_RAS_CE_REG 0x30105045#define SEC_RAS_FE_REG 0x30105446#define SEC_RAS_NFE_REG 0x30105847#define SEC_RAS_FE_ENB_MSK 0x048#define SEC_OOO_SHUTDOWN_SEL 0x30101449#define SEC_RAS_DISABLE 0x050#define SEC_MEM_START_INIT_REG 0x30110051#define SEC_MEM_INIT_DONE_REG 0x30110452 53/* clock gating */54#define SEC_CONTROL_REG 0x30120055#define SEC_DYNAMIC_GATE_REG 0x30121c56#define SEC_CORE_AUTO_GATE 0x30212c57#define SEC_DYNAMIC_GATE_EN 0x7fff58#define SEC_CORE_AUTO_GATE_EN GENMASK(3, 0)59#define SEC_CLK_GATE_ENABLE BIT(3)60#define SEC_CLK_GATE_DISABLE (~BIT(3))61 62#define SEC_TRNG_EN_SHIFT 863#define SEC_AXI_SHUTDOWN_ENABLE BIT(12)64#define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF65 66#define SEC_INTERFACE_USER_CTRL0_REG 0x30122067#define SEC_INTERFACE_USER_CTRL1_REG 0x30122468#define SEC_SAA_EN_REG 0x30127069#define SEC_BD_ERR_CHK_EN_REG0 0x30138070#define SEC_BD_ERR_CHK_EN_REG1 0x30138471#define SEC_BD_ERR_CHK_EN_REG3 0x30138c72 73#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15))74#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7))75#define SEC_USER1_ENABLE_CONTEXT_SSV BIT(24)76#define SEC_USER1_ENABLE_DATA_SSV BIT(16)77#define SEC_USER1_WB_CONTEXT_SSV BIT(8)78#define SEC_USER1_WB_DATA_SSV BIT(0)79#define SEC_USER1_SVA_SET (SEC_USER1_ENABLE_CONTEXT_SSV | \80 SEC_USER1_ENABLE_DATA_SSV | \81 SEC_USER1_WB_CONTEXT_SSV | \82 SEC_USER1_WB_DATA_SSV)83#define SEC_USER1_SMMU_SVA (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET)84#define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET)85#define SEC_INTERFACE_USER_CTRL0_REG_V3 0x30222086#define SEC_INTERFACE_USER_CTRL1_REG_V3 0x30222487#define SEC_USER1_SMMU_NORMAL_V3 (BIT(23) | BIT(17) | BIT(11) | BIT(5))88#define SEC_USER1_SMMU_MASK_V3 0xFF79E79E89#define SEC_CORE_INT_STATUS_M_ECC BIT(2)90 91#define SEC_PREFETCH_CFG 0x30113092#define SEC_SVA_TRANS 0x301EC493#define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11)))94#define SEC_PREFETCH_DISABLE BIT(1)95#define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11))96 97#define SEC_DELAY_10_US 1098#define SEC_POLL_TIMEOUT_US 100099#define SEC_DBGFS_VAL_MAX_LEN 20100#define SEC_SINGLE_PORT_MAX_TRANS 0x2060101 102#define SEC_SQE_MASK_OFFSET 16103#define SEC_SQE_MASK_LEN 108104#define SEC_SHAPER_TYPE_RATE 400105 106#define SEC_DFX_BASE 0x301000107#define SEC_DFX_CORE 0x302100108#define SEC_DFX_COMMON1 0x301600109#define SEC_DFX_COMMON2 0x301C00110#define SEC_DFX_BASE_LEN 0x9D111#define SEC_DFX_CORE_LEN 0x32B112#define SEC_DFX_COMMON1_LEN 0x45113#define SEC_DFX_COMMON2_LEN 0xBA114 115#define SEC_ALG_BITMAP_SHIFT 32116 117#define SEC_CIPHER_BITMAP (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \118 GENMASK(24, 21))119#define SEC_DIGEST_BITMAP (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \120 GENMASK_ULL(42, 25))121#define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \122 GENMASK_ULL(45, 43))123 124struct sec_hw_error {125 u32 int_msk;126 const char *msg;127};128 129struct sec_dfx_item {130 const char *name;131 u32 offset;132};133 134static const char sec_name[] = "hisi_sec2";135static struct dentry *sec_debugfs_root;136 137static struct hisi_qm_list sec_devices = {138 .register_to_crypto = sec_register_to_crypto,139 .unregister_from_crypto = sec_unregister_from_crypto,140};141 142static const struct hisi_qm_cap_info sec_basic_info[] = {143 {SEC_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77},144 {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77},145 {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77},146 {SEC_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},147 {SEC_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177},148 {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177},149 {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177},150 {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088},151 {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1},152 {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1},153 {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4},154 {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4},155 {SEC_CORE_ENABLE_BITMAP, 0x3140, 0, GENMASK(31, 0), 0x17F, 0x17F, 0xF},156 {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF},157 {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C},158 {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},159 {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},160 {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},161 {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},162 {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},163 {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},164 {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},165 {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},166 {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},167 {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF},168};169 170static const u32 sec_pre_store_caps[] = {171 SEC_DRV_ALG_BITMAP_LOW,172 SEC_DRV_ALG_BITMAP_HIGH,173 SEC_DEV_ALG_BITMAP_LOW,174 SEC_DEV_ALG_BITMAP_HIGH,175};176 177static const struct qm_dev_alg sec_dev_algs[] = { {178 .alg_msk = SEC_CIPHER_BITMAP,179 .alg = "cipher\n",180 }, {181 .alg_msk = SEC_DIGEST_BITMAP,182 .alg = "digest\n",183 }, {184 .alg_msk = SEC_AEAD_BITMAP,185 .alg = "aead\n",186 },187};188 189static const struct sec_hw_error sec_hw_errors[] = {190 {191 .int_msk = BIT(0),192 .msg = "sec_axi_rresp_err_rint"193 },194 {195 .int_msk = BIT(1),196 .msg = "sec_axi_bresp_err_rint"197 },198 {199 .int_msk = BIT(2),200 .msg = "sec_ecc_2bit_err_rint"201 },202 {203 .int_msk = BIT(3),204 .msg = "sec_ecc_1bit_err_rint"205 },206 {207 .int_msk = BIT(4),208 .msg = "sec_req_trng_timeout_rint"209 },210 {211 .int_msk = BIT(5),212 .msg = "sec_fsm_hbeat_rint"213 },214 {215 .int_msk = BIT(6),216 .msg = "sec_channel_req_rng_timeout_rint"217 },218 {219 .int_msk = BIT(7),220 .msg = "sec_bd_err_rint"221 },222 {223 .int_msk = BIT(8),224 .msg = "sec_chain_buff_err_rint"225 },226 {227 .int_msk = BIT(14),228 .msg = "sec_no_secure_access"229 },230 {231 .int_msk = BIT(15),232 .msg = "sec_wrapping_key_auth_err"233 },234 {235 .int_msk = BIT(16),236 .msg = "sec_km_key_crc_fail"237 },238 {239 .int_msk = BIT(17),240 .msg = "sec_axi_poison_err"241 },242 {243 .int_msk = BIT(18),244 .msg = "sec_sva_err"245 },246 {}247};248 249static const char * const sec_dbg_file_name[] = {250 [SEC_CLEAR_ENABLE] = "clear_enable",251};252 253static struct sec_dfx_item sec_dfx_labels[] = {254 {"send_cnt", offsetof(struct sec_dfx, send_cnt)},255 {"recv_cnt", offsetof(struct sec_dfx, recv_cnt)},256 {"send_busy_cnt", offsetof(struct sec_dfx, send_busy_cnt)},257 {"recv_busy_cnt", offsetof(struct sec_dfx, recv_busy_cnt)},258 {"err_bd_cnt", offsetof(struct sec_dfx, err_bd_cnt)},259 {"invalid_req_cnt", offsetof(struct sec_dfx, invalid_req_cnt)},260 {"done_flag_cnt", offsetof(struct sec_dfx, done_flag_cnt)},261};262 263static const struct debugfs_reg32 sec_dfx_regs[] = {264 {"SEC_PF_ABNORMAL_INT_SOURCE ", 0x301010},265 {"SEC_SAA_EN ", 0x301270},266 {"SEC_BD_LATENCY_MIN ", 0x301600},267 {"SEC_BD_LATENCY_MAX ", 0x301608},268 {"SEC_BD_LATENCY_AVG ", 0x30160C},269 {"SEC_BD_NUM_IN_SAA0 ", 0x301670},270 {"SEC_BD_NUM_IN_SAA1 ", 0x301674},271 {"SEC_BD_NUM_IN_SEC ", 0x301680},272 {"SEC_ECC_1BIT_CNT ", 0x301C00},273 {"SEC_ECC_1BIT_INFO ", 0x301C04},274 {"SEC_ECC_2BIT_CNT ", 0x301C10},275 {"SEC_ECC_2BIT_INFO ", 0x301C14},276 {"SEC_BD_SAA0 ", 0x301C20},277 {"SEC_BD_SAA1 ", 0x301C24},278 {"SEC_BD_SAA2 ", 0x301C28},279 {"SEC_BD_SAA3 ", 0x301C2C},280 {"SEC_BD_SAA4 ", 0x301C30},281 {"SEC_BD_SAA5 ", 0x301C34},282 {"SEC_BD_SAA6 ", 0x301C38},283 {"SEC_BD_SAA7 ", 0x301C3C},284 {"SEC_BD_SAA8 ", 0x301C40},285 {"SEC_RAS_CE_ENABLE ", 0x301050},286 {"SEC_RAS_FE_ENABLE ", 0x301054},287 {"SEC_RAS_NFE_ENABLE ", 0x301058},288 {"SEC_REQ_TRNG_TIME_TH ", 0x30112C},289 {"SEC_CHANNEL_RNG_REQ_THLD ", 0x302110},290};291 292/* define the SEC's dfx regs region and region length */293static struct dfx_diff_registers sec_diff_regs[] = {294 {295 .reg_offset = SEC_DFX_BASE,296 .reg_len = SEC_DFX_BASE_LEN,297 }, {298 .reg_offset = SEC_DFX_COMMON1,299 .reg_len = SEC_DFX_COMMON1_LEN,300 }, {301 .reg_offset = SEC_DFX_COMMON2,302 .reg_len = SEC_DFX_COMMON2_LEN,303 }, {304 .reg_offset = SEC_DFX_CORE,305 .reg_len = SEC_DFX_CORE_LEN,306 },307};308 309static int sec_diff_regs_show(struct seq_file *s, void *unused)310{311 struct hisi_qm *qm = s->private;312 313 hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,314 ARRAY_SIZE(sec_diff_regs));315 316 return 0;317}318DEFINE_SHOW_ATTRIBUTE(sec_diff_regs);319 320static bool pf_q_num_flag;321static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp)322{323 pf_q_num_flag = true;324 325 return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF);326}327 328static const struct kernel_param_ops sec_pf_q_num_ops = {329 .set = sec_pf_q_num_set,330 .get = param_get_int,331};332 333static u32 pf_q_num = SEC_PF_DEF_Q_NUM;334module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444);335MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");336 337static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp)338{339 u32 ctx_q_num;340 int ret;341 342 if (!val)343 return -EINVAL;344 345 ret = kstrtou32(val, 10, &ctx_q_num);346 if (ret)347 return -EINVAL;348 349 if (!ctx_q_num || ctx_q_num > SEC_CTX_Q_NUM_MAX || ctx_q_num & 0x1) {350 pr_err("ctx queue num[%u] is invalid!\n", ctx_q_num);351 return -EINVAL;352 }353 354 return param_set_int(val, kp);355}356 357static const struct kernel_param_ops sec_ctx_q_num_ops = {358 .set = sec_ctx_q_num_set,359 .get = param_get_int,360};361static u32 ctx_q_num = SEC_CTX_Q_NUM_DEF;362module_param_cb(ctx_q_num, &sec_ctx_q_num_ops, &ctx_q_num, 0444);363MODULE_PARM_DESC(ctx_q_num, "Queue num in ctx (2 default, 2, 4, ..., 32)");364 365static const struct kernel_param_ops vfs_num_ops = {366 .set = vfs_num_set,367 .get = param_get_int,368};369 370static u32 vfs_num;371module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);372MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");373 374void sec_destroy_qps(struct hisi_qp **qps, int qp_num)375{376 hisi_qm_free_qps(qps, qp_num);377 kfree(qps);378}379 380struct hisi_qp **sec_create_qps(void)381{382 int node = cpu_to_node(raw_smp_processor_id());383 u32 ctx_num = ctx_q_num;384 struct hisi_qp **qps;385 int ret;386 387 qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL);388 if (!qps)389 return NULL;390 391 ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps);392 if (!ret)393 return qps;394 395 kfree(qps);396 return NULL;397}398 399u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low)400{401 u32 cap_val_h, cap_val_l;402 403 cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val;404 cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val;405 406 return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l;407}408 409static const struct kernel_param_ops sec_uacce_mode_ops = {410 .set = uacce_mode_set,411 .get = param_get_int,412};413 414/*415 * uacce_mode = 0 means sec only register to crypto,416 * uacce_mode = 1 means sec both register to crypto and uacce.417 */418static u32 uacce_mode = UACCE_MODE_NOUACCE;419module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444);420MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);421 422static const struct pci_device_id sec_dev_ids[] = {423 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_PF) },424 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },425 { 0, }426};427MODULE_DEVICE_TABLE(pci, sec_dev_ids);428 429static void sec_set_endian(struct hisi_qm *qm)430{431 u32 reg;432 433 reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);434 reg &= ~(BIT(1) | BIT(0));435 if (!IS_ENABLED(CONFIG_64BIT))436 reg |= BIT(1);437 438 if (!IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))439 reg |= BIT(0);440 441 writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);442}443 444static void sec_engine_sva_config(struct hisi_qm *qm)445{446 u32 reg;447 448 if (qm->ver > QM_HW_V2) {449 reg = readl_relaxed(qm->io_base +450 SEC_INTERFACE_USER_CTRL0_REG_V3);451 reg |= SEC_USER0_SMMU_NORMAL;452 writel_relaxed(reg, qm->io_base +453 SEC_INTERFACE_USER_CTRL0_REG_V3);454 455 reg = readl_relaxed(qm->io_base +456 SEC_INTERFACE_USER_CTRL1_REG_V3);457 reg &= SEC_USER1_SMMU_MASK_V3;458 reg |= SEC_USER1_SMMU_NORMAL_V3;459 writel_relaxed(reg, qm->io_base +460 SEC_INTERFACE_USER_CTRL1_REG_V3);461 } else {462 reg = readl_relaxed(qm->io_base +463 SEC_INTERFACE_USER_CTRL0_REG);464 reg |= SEC_USER0_SMMU_NORMAL;465 writel_relaxed(reg, qm->io_base +466 SEC_INTERFACE_USER_CTRL0_REG);467 reg = readl_relaxed(qm->io_base +468 SEC_INTERFACE_USER_CTRL1_REG);469 reg &= SEC_USER1_SMMU_MASK;470 if (qm->use_sva)471 reg |= SEC_USER1_SMMU_SVA;472 else473 reg |= SEC_USER1_SMMU_NORMAL;474 writel_relaxed(reg, qm->io_base +475 SEC_INTERFACE_USER_CTRL1_REG);476 }477}478 479static void sec_open_sva_prefetch(struct hisi_qm *qm)480{481 u32 val;482 int ret;483 484 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))485 return;486 487 /* Enable prefetch */488 val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);489 val &= SEC_PREFETCH_ENABLE;490 writel(val, qm->io_base + SEC_PREFETCH_CFG);491 492 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG,493 val, !(val & SEC_PREFETCH_DISABLE),494 SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US);495 if (ret)496 pci_err(qm->pdev, "failed to open sva prefetch\n");497}498 499static void sec_close_sva_prefetch(struct hisi_qm *qm)500{501 u32 val;502 int ret;503 504 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))505 return;506 507 val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);508 val |= SEC_PREFETCH_DISABLE;509 writel(val, qm->io_base + SEC_PREFETCH_CFG);510 511 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS,512 val, !(val & SEC_SVA_DISABLE_READY),513 SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US);514 if (ret)515 pci_err(qm->pdev, "failed to close sva prefetch\n");516}517 518static void sec_enable_clock_gate(struct hisi_qm *qm)519{520 u32 val;521 522 if (qm->ver < QM_HW_V3)523 return;524 525 val = readl_relaxed(qm->io_base + SEC_CONTROL_REG);526 val |= SEC_CLK_GATE_ENABLE;527 writel_relaxed(val, qm->io_base + SEC_CONTROL_REG);528 529 val = readl(qm->io_base + SEC_DYNAMIC_GATE_REG);530 val |= SEC_DYNAMIC_GATE_EN;531 writel(val, qm->io_base + SEC_DYNAMIC_GATE_REG);532 533 val = readl(qm->io_base + SEC_CORE_AUTO_GATE);534 val |= SEC_CORE_AUTO_GATE_EN;535 writel(val, qm->io_base + SEC_CORE_AUTO_GATE);536}537 538static void sec_disable_clock_gate(struct hisi_qm *qm)539{540 u32 val;541 542 /* Kunpeng920 needs to close clock gating */543 val = readl_relaxed(qm->io_base + SEC_CONTROL_REG);544 val &= SEC_CLK_GATE_DISABLE;545 writel_relaxed(val, qm->io_base + SEC_CONTROL_REG);546}547 548static int sec_engine_init(struct hisi_qm *qm)549{550 int ret;551 u32 reg;552 553 /* disable clock gate control before mem init */554 sec_disable_clock_gate(qm);555 556 writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG);557 558 ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG,559 reg, reg & 0x1, SEC_DELAY_10_US,560 SEC_POLL_TIMEOUT_US);561 if (ret) {562 pci_err(qm->pdev, "fail to init sec mem\n");563 return ret;564 }565 566 reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);567 reg |= (0x1 << SEC_TRNG_EN_SHIFT);568 writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);569 570 sec_engine_sva_config(qm);571 572 writel(SEC_SINGLE_PORT_MAX_TRANS,573 qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS);574 575 reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver);576 writel(reg, qm->io_base + SEC_SAA_EN_REG);577 578 if (qm->ver < QM_HW_V3) {579 /* HW V2 enable sm4 extra mode, as ctr/ecb */580 writel_relaxed(SEC_BD_ERR_CHK_EN0,581 qm->io_base + SEC_BD_ERR_CHK_EN_REG0);582 583 /* HW V2 enable sm4 xts mode multiple iv */584 writel_relaxed(SEC_BD_ERR_CHK_EN1,585 qm->io_base + SEC_BD_ERR_CHK_EN_REG1);586 writel_relaxed(SEC_BD_ERR_CHK_EN3,587 qm->io_base + SEC_BD_ERR_CHK_EN_REG3);588 }589 590 /* config endian */591 sec_set_endian(qm);592 593 sec_enable_clock_gate(qm);594 595 return 0;596}597 598static int sec_set_user_domain_and_cache(struct hisi_qm *qm)599{600 /* qm user domain */601 writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1);602 writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE);603 writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1);604 writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE);605 writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE);606 607 /* qm cache */608 writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG);609 writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE);610 611 /* disable FLR triggered by BME(bus master enable) */612 writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG);613 writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);614 615 /* enable sqc,cqc writeback */616 writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |617 CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |618 FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL);619 620 return sec_engine_init(qm);621}622 623/* sec_debug_regs_clear() - clear the sec debug regs */624static void sec_debug_regs_clear(struct hisi_qm *qm)625{626 int i;627 628 /* clear sec dfx regs */629 writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE);630 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++)631 readl(qm->io_base + sec_dfx_regs[i].offset);632 633 /* clear rdclr_en */634 writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE);635 636 hisi_qm_debug_regs_clear(qm);637}638 639static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable)640{641 u32 val1, val2;642 643 val1 = readl(qm->io_base + SEC_CONTROL_REG);644 if (enable) {645 val1 |= SEC_AXI_SHUTDOWN_ENABLE;646 val2 = hisi_qm_get_hw_info(qm, sec_basic_info,647 SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);648 } else {649 val1 &= SEC_AXI_SHUTDOWN_DISABLE;650 val2 = 0x0;651 }652 653 if (qm->ver > QM_HW_V2)654 writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL);655 656 writel(val1, qm->io_base + SEC_CONTROL_REG);657}658 659static void sec_hw_error_enable(struct hisi_qm *qm)660{661 u32 ce, nfe;662 663 if (qm->ver == QM_HW_V1) {664 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);665 pci_info(qm->pdev, "V1 not support hw error handle\n");666 return;667 }668 669 ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver);670 nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver);671 672 /* clear SEC hw error source if having */673 writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE);674 675 /* enable RAS int */676 writel(ce, qm->io_base + SEC_RAS_CE_REG);677 writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG);678 writel(nfe, qm->io_base + SEC_RAS_NFE_REG);679 680 /* enable SEC block master OOO when nfe occurs on Kunpeng930 */681 sec_master_ooo_ctrl(qm, true);682 683 /* enable SEC hw error interrupts */684 writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK);685}686 687static void sec_hw_error_disable(struct hisi_qm *qm)688{689 /* disable SEC hw error interrupts */690 writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);691 692 /* disable SEC block master OOO when nfe occurs on Kunpeng930 */693 sec_master_ooo_ctrl(qm, false);694 695 /* disable RAS int */696 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);697 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG);698 writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG);699}700 701static u32 sec_clear_enable_read(struct hisi_qm *qm)702{703 return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &704 SEC_CTRL_CNT_CLR_CE_BIT;705}706 707static int sec_clear_enable_write(struct hisi_qm *qm, u32 val)708{709 u32 tmp;710 711 if (val != 1 && val)712 return -EINVAL;713 714 tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &715 ~SEC_CTRL_CNT_CLR_CE_BIT) | val;716 writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE);717 718 return 0;719}720 721static ssize_t sec_debug_read(struct file *filp, char __user *buf,722 size_t count, loff_t *pos)723{724 struct sec_debug_file *file = filp->private_data;725 char tbuf[SEC_DBGFS_VAL_MAX_LEN];726 struct hisi_qm *qm = file->qm;727 u32 val;728 int ret;729 730 ret = hisi_qm_get_dfx_access(qm);731 if (ret)732 return ret;733 734 spin_lock_irq(&file->lock);735 736 switch (file->index) {737 case SEC_CLEAR_ENABLE:738 val = sec_clear_enable_read(qm);739 break;740 default:741 goto err_input;742 }743 744 spin_unlock_irq(&file->lock);745 746 hisi_qm_put_dfx_access(qm);747 ret = snprintf(tbuf, SEC_DBGFS_VAL_MAX_LEN, "%u\n", val);748 return simple_read_from_buffer(buf, count, pos, tbuf, ret);749 750err_input:751 spin_unlock_irq(&file->lock);752 hisi_qm_put_dfx_access(qm);753 return -EINVAL;754}755 756static ssize_t sec_debug_write(struct file *filp, const char __user *buf,757 size_t count, loff_t *pos)758{759 struct sec_debug_file *file = filp->private_data;760 char tbuf[SEC_DBGFS_VAL_MAX_LEN];761 struct hisi_qm *qm = file->qm;762 unsigned long val;763 int len, ret;764 765 if (*pos != 0)766 return 0;767 768 if (count >= SEC_DBGFS_VAL_MAX_LEN)769 return -ENOSPC;770 771 len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1,772 pos, buf, count);773 if (len < 0)774 return len;775 776 tbuf[len] = '\0';777 if (kstrtoul(tbuf, 0, &val))778 return -EFAULT;779 780 ret = hisi_qm_get_dfx_access(qm);781 if (ret)782 return ret;783 784 spin_lock_irq(&file->lock);785 786 switch (file->index) {787 case SEC_CLEAR_ENABLE:788 ret = sec_clear_enable_write(qm, val);789 if (ret)790 goto err_input;791 break;792 default:793 ret = -EINVAL;794 goto err_input;795 }796 797 ret = count;798 799 err_input:800 spin_unlock_irq(&file->lock);801 hisi_qm_put_dfx_access(qm);802 return ret;803}804 805static const struct file_operations sec_dbg_fops = {806 .owner = THIS_MODULE,807 .open = simple_open,808 .read = sec_debug_read,809 .write = sec_debug_write,810};811 812static int sec_debugfs_atomic64_get(void *data, u64 *val)813{814 *val = atomic64_read((atomic64_t *)data);815 816 return 0;817}818 819static int sec_debugfs_atomic64_set(void *data, u64 val)820{821 if (val)822 return -EINVAL;823 824 atomic64_set((atomic64_t *)data, 0);825 826 return 0;827}828 829DEFINE_DEBUGFS_ATTRIBUTE(sec_atomic64_ops, sec_debugfs_atomic64_get,830 sec_debugfs_atomic64_set, "%lld\n");831 832static int sec_regs_show(struct seq_file *s, void *unused)833{834 hisi_qm_regs_dump(s, s->private);835 836 return 0;837}838 839DEFINE_SHOW_ATTRIBUTE(sec_regs);840 841static int sec_core_debug_init(struct hisi_qm *qm)842{843 struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs;844 struct sec_dev *sec = container_of(qm, struct sec_dev, qm);845 struct device *dev = &qm->pdev->dev;846 struct sec_dfx *dfx = &sec->debug.dfx;847 struct debugfs_regset32 *regset;848 struct dentry *tmp_d;849 int i;850 851 tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root);852 853 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);854 if (!regset)855 return -ENOMEM;856 857 regset->regs = sec_dfx_regs;858 regset->nregs = ARRAY_SIZE(sec_dfx_regs);859 regset->base = qm->io_base;860 regset->dev = dev;861 862 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF)863 debugfs_create_file("regs", 0444, tmp_d, regset, &sec_regs_fops);864 if (qm->fun_type == QM_HW_PF && sec_regs)865 debugfs_create_file("diff_regs", 0444, tmp_d,866 qm, &sec_diff_regs_fops);867 868 for (i = 0; i < ARRAY_SIZE(sec_dfx_labels); i++) {869 atomic64_t *data = (atomic64_t *)((uintptr_t)dfx +870 sec_dfx_labels[i].offset);871 debugfs_create_file(sec_dfx_labels[i].name, 0644,872 tmp_d, data, &sec_atomic64_ops);873 }874 875 return 0;876}877 878static int sec_debug_init(struct hisi_qm *qm)879{880 struct sec_dev *sec = container_of(qm, struct sec_dev, qm);881 int i;882 883 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) {884 for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) {885 spin_lock_init(&sec->debug.files[i].lock);886 sec->debug.files[i].index = i;887 sec->debug.files[i].qm = qm;888 889 debugfs_create_file(sec_dbg_file_name[i], 0600,890 qm->debug.debug_root,891 sec->debug.files + i,892 &sec_dbg_fops);893 }894 }895 896 return sec_core_debug_init(qm);897}898 899static int sec_debugfs_init(struct hisi_qm *qm)900{901 struct device *dev = &qm->pdev->dev;902 int ret;903 904 ret = hisi_qm_regs_debugfs_init(qm, sec_diff_regs, ARRAY_SIZE(sec_diff_regs));905 if (ret) {906 dev_warn(dev, "Failed to init SEC diff regs!\n");907 return ret;908 }909 910 qm->debug.debug_root = debugfs_create_dir(dev_name(dev),911 sec_debugfs_root);912 qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET;913 qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN;914 915 hisi_qm_debug_init(qm);916 917 ret = sec_debug_init(qm);918 if (ret)919 goto debugfs_remove;920 921 return 0;922 923debugfs_remove:924 debugfs_remove_recursive(qm->debug.debug_root);925 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs));926 return ret;927}928 929static void sec_debugfs_exit(struct hisi_qm *qm)930{931 debugfs_remove_recursive(qm->debug.debug_root);932 933 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(sec_diff_regs));934}935 936static int sec_show_last_regs_init(struct hisi_qm *qm)937{938 struct qm_debug *debug = &qm->debug;939 int i;940 941 debug->last_words = kcalloc(ARRAY_SIZE(sec_dfx_regs),942 sizeof(unsigned int), GFP_KERNEL);943 if (!debug->last_words)944 return -ENOMEM;945 946 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++)947 debug->last_words[i] = readl_relaxed(qm->io_base +948 sec_dfx_regs[i].offset);949 950 return 0;951}952 953static void sec_show_last_regs_uninit(struct hisi_qm *qm)954{955 struct qm_debug *debug = &qm->debug;956 957 if (qm->fun_type == QM_HW_VF || !debug->last_words)958 return;959 960 kfree(debug->last_words);961 debug->last_words = NULL;962}963 964static void sec_show_last_dfx_regs(struct hisi_qm *qm)965{966 struct qm_debug *debug = &qm->debug;967 struct pci_dev *pdev = qm->pdev;968 u32 val;969 int i;970 971 if (qm->fun_type == QM_HW_VF || !debug->last_words)972 return;973 974 /* dumps last word of the debugging registers during controller reset */975 for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++) {976 val = readl_relaxed(qm->io_base + sec_dfx_regs[i].offset);977 if (val != debug->last_words[i])978 pci_info(pdev, "%s \t= 0x%08x => 0x%08x\n",979 sec_dfx_regs[i].name, debug->last_words[i], val);980 }981}982 983static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts)984{985 const struct sec_hw_error *errs = sec_hw_errors;986 struct device *dev = &qm->pdev->dev;987 u32 err_val;988 989 while (errs->msg) {990 if (errs->int_msk & err_sts) {991 dev_err(dev, "%s [error status=0x%x] found\n",992 errs->msg, errs->int_msk);993 994 if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) {995 err_val = readl(qm->io_base +996 SEC_CORE_SRAM_ECC_ERR_INFO);997 dev_err(dev, "multi ecc sram num=0x%x\n",998 ((err_val) >> SEC_ECC_NUM) &999 SEC_ECC_MASH);1000 }1001 }1002 errs++;1003 }1004}1005 1006static u32 sec_get_hw_err_status(struct hisi_qm *qm)1007{1008 return readl(qm->io_base + SEC_CORE_INT_STATUS);1009}1010 1011static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)1012{1013 u32 nfe;1014 1015 writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE);1016 nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver);1017 writel(nfe, qm->io_base + SEC_RAS_NFE_REG);1018}1019 1020static void sec_open_axi_master_ooo(struct hisi_qm *qm)1021{1022 u32 val;1023 1024 val = readl(qm->io_base + SEC_CONTROL_REG);1025 writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG);1026 writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG);1027}1028 1029static void sec_err_info_init(struct hisi_qm *qm)1030{1031 struct hisi_qm_err_info *err_info = &qm->err_info;1032 1033 err_info->fe = SEC_RAS_FE_ENB_MSK;1034 err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver);1035 err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver);1036 err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC;1037 err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info,1038 SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);1039 err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info,1040 SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);1041 err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info,1042 SEC_QM_RESET_MASK_CAP, qm->cap_ver);1043 err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info,1044 SEC_RESET_MASK_CAP, qm->cap_ver);1045 err_info->msi_wr_port = BIT(0);1046 err_info->acpi_rst = "SRST";1047}1048 1049static const struct hisi_qm_err_ini sec_err_ini = {1050 .hw_init = sec_set_user_domain_and_cache,1051 .hw_err_enable = sec_hw_error_enable,1052 .hw_err_disable = sec_hw_error_disable,1053 .get_dev_hw_err_status = sec_get_hw_err_status,1054 .clear_dev_hw_err_status = sec_clear_hw_err_status,1055 .log_dev_hw_err = sec_log_hw_error,1056 .open_axi_master_ooo = sec_open_axi_master_ooo,1057 .open_sva_prefetch = sec_open_sva_prefetch,1058 .close_sva_prefetch = sec_close_sva_prefetch,1059 .show_last_dfx_regs = sec_show_last_dfx_regs,1060 .err_info_init = sec_err_info_init,1061};1062 1063static int sec_pf_probe_init(struct sec_dev *sec)1064{1065 struct hisi_qm *qm = &sec->qm;1066 int ret;1067 1068 ret = sec_set_user_domain_and_cache(qm);1069 if (ret)1070 return ret;1071 1072 sec_open_sva_prefetch(qm);1073 hisi_qm_dev_err_init(qm);1074 sec_debug_regs_clear(qm);1075 ret = sec_show_last_regs_init(qm);1076 if (ret)1077 pci_err(qm->pdev, "Failed to init last word regs!\n");1078 1079 return ret;1080}1081 1082static int sec_pre_store_cap_reg(struct hisi_qm *qm)1083{1084 struct hisi_qm_cap_record *sec_cap;1085 struct pci_dev *pdev = qm->pdev;1086 size_t i, size;1087 1088 size = ARRAY_SIZE(sec_pre_store_caps);1089 sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL);1090 if (!sec_cap)1091 return -ENOMEM;1092 1093 for (i = 0; i < size; i++) {1094 sec_cap[i].type = sec_pre_store_caps[i];1095 sec_cap[i].cap_val = hisi_qm_get_hw_info(qm, sec_basic_info,1096 sec_pre_store_caps[i], qm->cap_ver);1097 }1098 1099 qm->cap_tables.dev_cap_table = sec_cap;1100 1101 return 0;1102}1103 1104static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)1105{1106 u64 alg_msk;1107 int ret;1108 1109 qm->pdev = pdev;1110 qm->ver = pdev->revision;1111 qm->mode = uacce_mode;1112 qm->sqe_size = SEC_SQE_SIZE;1113 qm->dev_name = sec_name;1114 1115 qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) ?1116 QM_HW_PF : QM_HW_VF;1117 if (qm->fun_type == QM_HW_PF) {1118 qm->qp_base = SEC_PF_DEF_Q_BASE;1119 qm->qp_num = pf_q_num;1120 qm->debug.curr_qm_qp_num = pf_q_num;1121 qm->qm_list = &sec_devices;1122 qm->err_ini = &sec_err_ini;1123 if (pf_q_num_flag)1124 set_bit(QM_MODULE_PARAM, &qm->misc_ctl);1125 } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {1126 /*1127 * have no way to get qm configure in VM in v1 hardware,1128 * so currently force PF to uses SEC_PF_DEF_Q_NUM, and force1129 * to trigger only one VF in v1 hardware.1130 * v2 hardware has no such problem.1131 */1132 qm->qp_base = SEC_PF_DEF_Q_NUM;1133 qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;1134 }1135 1136 ret = hisi_qm_init(qm);1137 if (ret) {1138 pci_err(qm->pdev, "Failed to init sec qm configures!\n");1139 return ret;1140 }1141 1142 /* Fetch and save the value of capability registers */1143 ret = sec_pre_store_cap_reg(qm);1144 if (ret) {1145 pci_err(qm->pdev, "Failed to pre-store capability registers!\n");1146 hisi_qm_uninit(qm);1147 return ret;1148 }1149 1150 alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH_IDX, SEC_DEV_ALG_BITMAP_LOW_IDX);1151 ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs));1152 if (ret) {1153 pci_err(qm->pdev, "Failed to set sec algs!\n");1154 hisi_qm_uninit(qm);1155 }1156 1157 return ret;1158}1159 1160static void sec_qm_uninit(struct hisi_qm *qm)1161{1162 hisi_qm_uninit(qm);1163}1164 1165static int sec_probe_init(struct sec_dev *sec)1166{1167 u32 type_rate = SEC_SHAPER_TYPE_RATE;1168 struct hisi_qm *qm = &sec->qm;1169 int ret;1170 1171 if (qm->fun_type == QM_HW_PF) {1172 ret = sec_pf_probe_init(sec);1173 if (ret)1174 return ret;1175 /* enable shaper type 0 */1176 if (qm->ver >= QM_HW_V3) {1177 type_rate |= QM_SHAPER_ENABLE;1178 qm->type_rate = type_rate;1179 }1180 }1181 1182 return 0;1183}1184 1185static void sec_probe_uninit(struct hisi_qm *qm)1186{1187 if (qm->fun_type == QM_HW_VF)1188 return;1189 1190 sec_debug_regs_clear(qm);1191 sec_show_last_regs_uninit(qm);1192 sec_close_sva_prefetch(qm);1193 hisi_qm_dev_err_uninit(qm);1194}1195 1196static void sec_iommu_used_check(struct sec_dev *sec)1197{1198 struct iommu_domain *domain;1199 struct device *dev = &sec->qm.pdev->dev;1200 1201 domain = iommu_get_domain_for_dev(dev);1202 1203 /* Check if iommu is used */1204 sec->iommu_used = false;1205 if (domain) {1206 if (domain->type & __IOMMU_DOMAIN_PAGING)1207 sec->iommu_used = true;1208 dev_info(dev, "SMMU Opened, the iommu type = %u\n",1209 domain->type);1210 }1211}1212 1213static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)1214{1215 struct sec_dev *sec;1216 struct hisi_qm *qm;1217 int ret;1218 1219 sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL);1220 if (!sec)1221 return -ENOMEM;1222 1223 qm = &sec->qm;1224 ret = sec_qm_init(qm, pdev);1225 if (ret) {1226 pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret);1227 return ret;1228 }1229 1230 sec->ctx_q_num = ctx_q_num;1231 sec_iommu_used_check(sec);1232 1233 ret = sec_probe_init(sec);1234 if (ret) {1235 pci_err(pdev, "Failed to probe!\n");1236 goto err_qm_uninit;1237 }1238 1239 ret = hisi_qm_start(qm);1240 if (ret) {1241 pci_err(pdev, "Failed to start sec qm!\n");1242 goto err_probe_uninit;1243 }1244 1245 ret = sec_debugfs_init(qm);1246 if (ret)1247 pci_warn(pdev, "Failed to init debugfs!\n");1248 1249 hisi_qm_add_list(qm, &sec_devices);1250 ret = hisi_qm_alg_register(qm, &sec_devices, ctx_q_num);1251 if (ret < 0) {1252 pr_err("Failed to register driver to crypto.\n");1253 goto err_qm_del_list;1254 }1255 1256 if (qm->uacce) {1257 ret = uacce_register(qm->uacce);1258 if (ret) {1259 pci_err(pdev, "failed to register uacce (%d)!\n", ret);1260 goto err_alg_unregister;1261 }1262 }1263 1264 if (qm->fun_type == QM_HW_PF && vfs_num) {1265 ret = hisi_qm_sriov_enable(pdev, vfs_num);1266 if (ret < 0)1267 goto err_alg_unregister;1268 }1269 1270 hisi_qm_pm_init(qm);1271 1272 return 0;1273 1274err_alg_unregister:1275 hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num);1276err_qm_del_list:1277 hisi_qm_del_list(qm, &sec_devices);1278 sec_debugfs_exit(qm);1279 hisi_qm_stop(qm, QM_NORMAL);1280err_probe_uninit:1281 sec_probe_uninit(qm);1282err_qm_uninit:1283 sec_qm_uninit(qm);1284 return ret;1285}1286 1287static void sec_remove(struct pci_dev *pdev)1288{1289 struct hisi_qm *qm = pci_get_drvdata(pdev);1290 1291 hisi_qm_pm_uninit(qm);1292 hisi_qm_wait_task_finish(qm, &sec_devices);1293 hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num);1294 hisi_qm_del_list(qm, &sec_devices);1295 1296 if (qm->fun_type == QM_HW_PF && qm->vfs_num)1297 hisi_qm_sriov_disable(pdev, true);1298 1299 sec_debugfs_exit(qm);1300 1301 (void)hisi_qm_stop(qm, QM_NORMAL);1302 sec_probe_uninit(qm);1303 1304 sec_qm_uninit(qm);1305}1306 1307static const struct dev_pm_ops sec_pm_ops = {1308 SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)1309};1310 1311static const struct pci_error_handlers sec_err_handler = {1312 .error_detected = hisi_qm_dev_err_detected,1313 .slot_reset = hisi_qm_dev_slot_reset,1314 .reset_prepare = hisi_qm_reset_prepare,1315 .reset_done = hisi_qm_reset_done,1316};1317 1318static struct pci_driver sec_pci_driver = {1319 .name = "hisi_sec2",1320 .id_table = sec_dev_ids,1321 .probe = sec_probe,1322 .remove = sec_remove,1323 .err_handler = &sec_err_handler,1324 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?1325 hisi_qm_sriov_configure : NULL,1326 .shutdown = hisi_qm_dev_shutdown,1327 .driver.pm = &sec_pm_ops,1328};1329 1330struct pci_driver *hisi_sec_get_pf_driver(void)1331{1332 return &sec_pci_driver;1333}1334EXPORT_SYMBOL_GPL(hisi_sec_get_pf_driver);1335 1336static void sec_register_debugfs(void)1337{1338 if (!debugfs_initialized())1339 return;1340 1341 sec_debugfs_root = debugfs_create_dir("hisi_sec2", NULL);1342}1343 1344static void sec_unregister_debugfs(void)1345{1346 debugfs_remove_recursive(sec_debugfs_root);1347}1348 1349static int __init sec_init(void)1350{1351 int ret;1352 1353 hisi_qm_init_list(&sec_devices);1354 sec_register_debugfs();1355 1356 ret = pci_register_driver(&sec_pci_driver);1357 if (ret < 0) {1358 sec_unregister_debugfs();1359 pr_err("Failed to register pci driver.\n");1360 return ret;1361 }1362 1363 return 0;1364}1365 1366static void __exit sec_exit(void)1367{1368 pci_unregister_driver(&sec_pci_driver);1369 sec_unregister_debugfs();1370}1371 1372module_init(sec_init);1373module_exit(sec_exit);1374 1375MODULE_LICENSE("GPL v2");1376MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");1377MODULE_AUTHOR("Longfang Liu <liulongfang@huawei.com>");1378MODULE_AUTHOR("Kai Ye <yekai13@huawei.com>");1379MODULE_AUTHOR("Wei Zhang <zhangwei375@huawei.com>");1380MODULE_DESCRIPTION("Driver for HiSilicon SEC accelerator");1381