1599 lines · c
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)2/* Copyright(c) 2014 - 2020 Intel Corporation */3#include <linux/slab.h>4#include <linux/delay.h>5#include <linux/pci_ids.h>6 7#include "adf_accel_devices.h"8#include "adf_common_drv.h"9#include "icp_qat_hal.h"10#include "icp_qat_uclo.h"11 12#define BAD_REGADDR 0xffff13#define MAX_RETRY_TIMES 1000014#define INIT_CTX_ARB_VALUE 0x015#define INIT_CTX_ENABLE_VALUE 0x016#define INIT_PC_VALUE 0x017#define INIT_WAKEUP_EVENTS_VALUE 0x118#define INIT_SIG_EVENTS_VALUE 0x119#define INIT_CCENABLE_VALUE 0x200020#define RST_CSR_QAT_LSB 2021#define RST_CSR_AE_LSB 022#define MC_TIMESTAMP_ENABLE (0x1 << 7)23 24#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \25 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \26 (~(1 << CE_REG_PAR_ERR_BITPOS)))27#define INSERT_IMMED_GPRA_CONST(inst, const_val) \28 (inst = ((inst & 0xFFFF00C03FFull) | \29 ((((const_val) << 12) & 0x0FF00000ull) | \30 (((const_val) << 10) & 0x0003FC00ull))))31#define INSERT_IMMED_GPRB_CONST(inst, const_val) \32 (inst = ((inst & 0xFFFF00FFF00ull) | \33 ((((const_val) << 12) & 0x0FF00000ull) | \34 (((const_val) << 0) & 0x000000FFull))))35 36#define AE(handle, ae) ((handle)->hal_handle->aes[ae])37 38static const u64 inst_4b[] = {39 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,40 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,41 0x0A021000000ull42};43 44static const u64 inst[] = {45 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,46 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,47 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,48 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,49 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,50 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,51 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,52 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,53 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,54 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,55 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,56 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,57 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,58 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,59 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,60 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,61 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,62 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,63 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,64 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,65 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,66 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,67};68 69void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,70 unsigned char ae, unsigned int ctx_mask)71{72 AE(handle, ae).live_ctx_mask = ctx_mask;73}74 75#define CSR_RETRY_TIMES 50076static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,77 unsigned char ae, unsigned int csr)78{79 unsigned int iterations = CSR_RETRY_TIMES;80 int value;81 82 do {83 value = GET_AE_CSR(handle, ae, csr);84 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))85 return value;86 } while (iterations--);87 88 pr_err("QAT: Read CSR timeout\n");89 return 0;90}91 92static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,93 unsigned char ae, unsigned int csr,94 unsigned int value)95{96 unsigned int iterations = CSR_RETRY_TIMES;97 98 do {99 SET_AE_CSR(handle, ae, csr, value);100 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))101 return 0;102 } while (iterations--);103 104 pr_err("QAT: Write CSR Timeout\n");105 return -EFAULT;106}107 108static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,109 unsigned char ae, unsigned char ctx,110 unsigned int *events)111{112 unsigned int cur_ctx;113 114 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);115 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);116 *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);117 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);118}119 120static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,121 unsigned char ae, unsigned int cycles,122 int chk_inactive)123{124 unsigned int base_cnt = 0, cur_cnt = 0;125 unsigned int csr = (1 << ACS_ABO_BITPOS);126 int times = MAX_RETRY_TIMES;127 int elapsed_cycles = 0;128 129 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);130 base_cnt &= 0xffff;131 while ((int)cycles > elapsed_cycles && times--) {132 if (chk_inactive)133 csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);134 135 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);136 cur_cnt &= 0xffff;137 elapsed_cycles = cur_cnt - base_cnt;138 139 if (elapsed_cycles < 0)140 elapsed_cycles += 0x10000;141 142 /* ensure at least 8 time cycles elapsed in wait_cycles */143 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))144 return 0;145 }146 if (times < 0) {147 pr_err("QAT: wait_num_cycles time out\n");148 return -EFAULT;149 }150 return 0;151}152 153#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))154#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))155 156int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,157 unsigned char ae, unsigned char mode)158{159 unsigned int csr, new_csr;160 161 if (mode != 4 && mode != 8) {162 pr_err("QAT: bad ctx mode=%d\n", mode);163 return -EINVAL;164 }165 166 /* Sets the accelaration engine context mode to either four or eight */167 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);168 csr = IGNORE_W1C_MASK & csr;169 new_csr = (mode == 4) ?170 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :171 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);172 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);173 return 0;174}175 176int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,177 unsigned char ae, unsigned char mode)178{179 unsigned int csr, new_csr;180 181 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);182 csr &= IGNORE_W1C_MASK;183 184 new_csr = (mode) ?185 SET_BIT(csr, CE_NN_MODE_BITPOS) :186 CLR_BIT(csr, CE_NN_MODE_BITPOS);187 188 if (new_csr != csr)189 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);190 191 return 0;192}193 194int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,195 unsigned char ae, enum icp_qat_uof_regtype lm_type,196 unsigned char mode)197{198 unsigned int csr, new_csr;199 200 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);201 csr &= IGNORE_W1C_MASK;202 switch (lm_type) {203 case ICP_LMEM0:204 new_csr = (mode) ?205 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :206 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);207 break;208 case ICP_LMEM1:209 new_csr = (mode) ?210 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :211 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);212 break;213 case ICP_LMEM2:214 new_csr = (mode) ?215 SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :216 CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);217 break;218 case ICP_LMEM3:219 new_csr = (mode) ?220 SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :221 CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);222 break;223 default:224 pr_err("QAT: lmType = 0x%x\n", lm_type);225 return -EINVAL;226 }227 228 if (new_csr != csr)229 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);230 return 0;231}232 233void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,234 unsigned char ae, unsigned char mode)235{236 unsigned int csr, new_csr;237 238 csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);239 csr &= IGNORE_W1C_MASK;240 new_csr = (mode) ?241 SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :242 CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);243 if (new_csr != csr)244 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);245}246 247static unsigned short qat_hal_get_reg_addr(unsigned int type,248 unsigned short reg_num)249{250 unsigned short reg_addr;251 252 switch (type) {253 case ICP_GPA_ABS:254 case ICP_GPB_ABS:255 reg_addr = 0x80 | (reg_num & 0x7f);256 break;257 case ICP_GPA_REL:258 case ICP_GPB_REL:259 reg_addr = reg_num & 0x1f;260 break;261 case ICP_SR_RD_REL:262 case ICP_SR_WR_REL:263 case ICP_SR_REL:264 reg_addr = 0x180 | (reg_num & 0x1f);265 break;266 case ICP_SR_ABS:267 reg_addr = 0x140 | ((reg_num & 0x3) << 1);268 break;269 case ICP_DR_RD_REL:270 case ICP_DR_WR_REL:271 case ICP_DR_REL:272 reg_addr = 0x1c0 | (reg_num & 0x1f);273 break;274 case ICP_DR_ABS:275 reg_addr = 0x100 | ((reg_num & 0x3) << 1);276 break;277 case ICP_NEIGH_REL:278 reg_addr = 0x280 | (reg_num & 0x1f);279 break;280 case ICP_LMEM0:281 reg_addr = 0x200;282 break;283 case ICP_LMEM1:284 reg_addr = 0x220;285 break;286 case ICP_LMEM2:287 reg_addr = 0x2c0;288 break;289 case ICP_LMEM3:290 reg_addr = 0x2e0;291 break;292 case ICP_NO_DEST:293 reg_addr = 0x300 | (reg_num & 0xff);294 break;295 default:296 reg_addr = BAD_REGADDR;297 break;298 }299 return reg_addr;300}301 302void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)303{304 unsigned int reset_mask = handle->chip_info->icp_rst_mask;305 unsigned int reset_csr = handle->chip_info->icp_rst_csr;306 unsigned int csr_val;307 308 csr_val = GET_CAP_CSR(handle, reset_csr);309 csr_val |= reset_mask;310 SET_CAP_CSR(handle, reset_csr, csr_val);311}312 313static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,314 unsigned char ae, unsigned int ctx_mask,315 unsigned int ae_csr, unsigned int csr_val)316{317 unsigned int ctx, cur_ctx;318 319 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);320 321 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {322 if (!(ctx_mask & (1 << ctx)))323 continue;324 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);325 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);326 }327 328 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);329}330 331static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,332 unsigned char ae, unsigned char ctx,333 unsigned int ae_csr)334{335 unsigned int cur_ctx, csr_val;336 337 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);338 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);339 csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);340 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);341 342 return csr_val;343}344 345static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,346 unsigned char ae, unsigned int ctx_mask,347 unsigned int events)348{349 unsigned int ctx, cur_ctx;350 351 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);352 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {353 if (!(ctx_mask & (1 << ctx)))354 continue;355 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);356 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);357 }358 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);359}360 361static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,362 unsigned char ae, unsigned int ctx_mask,363 unsigned int events)364{365 unsigned int ctx, cur_ctx;366 367 cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);368 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {369 if (!(ctx_mask & (1 << ctx)))370 continue;371 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);372 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,373 events);374 }375 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);376}377 378static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)379{380 unsigned long ae_mask = handle->hal_handle->ae_mask;381 unsigned int base_cnt, cur_cnt;382 unsigned char ae;383 int times = MAX_RETRY_TIMES;384 385 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {386 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);387 base_cnt &= 0xffff;388 389 do {390 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);391 cur_cnt &= 0xffff;392 } while (times-- && (cur_cnt == base_cnt));393 394 if (times < 0) {395 pr_err("QAT: AE%d is inactive!!\n", ae);396 return -EFAULT;397 }398 }399 400 return 0;401}402 403int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,404 unsigned int ae)405{406 unsigned int enable = 0, active = 0;407 408 enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);409 active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);410 if ((enable & (0xff << CE_ENABLE_BITPOS)) ||411 (active & (1 << ACS_ABO_BITPOS)))412 return 1;413 else414 return 0;415}416 417static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)418{419 unsigned long ae_mask = handle->hal_handle->ae_mask;420 unsigned int misc_ctl_csr, misc_ctl;421 unsigned char ae;422 423 misc_ctl_csr = handle->chip_info->misc_ctl_csr;424 /* stop the timestamp timers */425 misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr);426 if (misc_ctl & MC_TIMESTAMP_ENABLE)427 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl &428 (~MC_TIMESTAMP_ENABLE));429 430 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {431 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);432 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);433 }434 /* start timestamp timers */435 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);436}437 438#define ESRAM_AUTO_TINIT BIT(2)439#define ESRAM_AUTO_TINIT_DONE BIT(3)440#define ESRAM_AUTO_INIT_USED_CYCLES (1640)441#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C442static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)443{444 void __iomem *csr_addr =445 (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +446 ESRAM_AUTO_INIT_CSR_OFFSET);447 unsigned int csr_val;448 int times = 30;449 450 if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)451 return 0;452 453 csr_val = ADF_CSR_RD(csr_addr, 0);454 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))455 return 0;456 457 csr_val = ADF_CSR_RD(csr_addr, 0);458 csr_val |= ESRAM_AUTO_TINIT;459 ADF_CSR_WR(csr_addr, 0, csr_val);460 461 do {462 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);463 csr_val = ADF_CSR_RD(csr_addr, 0);464 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);465 if (times < 0) {466 pr_err("QAT: Fail to init eSram!\n");467 return -EFAULT;468 }469 return 0;470}471 472#define SHRAM_INIT_CYCLES 2060473int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)474{475 unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr;476 unsigned int reset_mask = handle->chip_info->icp_rst_mask;477 unsigned int reset_csr = handle->chip_info->icp_rst_csr;478 unsigned long ae_mask = handle->hal_handle->ae_mask;479 unsigned char ae = 0;480 unsigned int times = 100;481 unsigned int csr_val;482 483 /* write to the reset csr */484 csr_val = GET_CAP_CSR(handle, reset_csr);485 csr_val &= ~reset_mask;486 do {487 SET_CAP_CSR(handle, reset_csr, csr_val);488 if (!(times--))489 goto out_err;490 csr_val = GET_CAP_CSR(handle, reset_csr);491 csr_val &= reset_mask;492 } while (csr_val);493 /* enable clock */494 csr_val = GET_CAP_CSR(handle, clk_csr);495 csr_val |= reset_mask;496 SET_CAP_CSR(handle, clk_csr, csr_val);497 if (qat_hal_check_ae_alive(handle))498 goto out_err;499 500 /* Set undefined power-up/reset states to reasonable default values */501 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {502 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,503 INIT_CTX_ENABLE_VALUE);504 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,505 CTX_STS_INDIRECT,506 handle->hal_handle->upc_mask &507 INIT_PC_VALUE);508 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);509 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);510 qat_hal_put_wakeup_event(handle, ae,511 ICP_QAT_UCLO_AE_ALL_CTX,512 INIT_WAKEUP_EVENTS_VALUE);513 qat_hal_put_sig_event(handle, ae,514 ICP_QAT_UCLO_AE_ALL_CTX,515 INIT_SIG_EVENTS_VALUE);516 }517 if (qat_hal_init_esram(handle))518 goto out_err;519 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))520 goto out_err;521 qat_hal_reset_timestamp(handle);522 523 return 0;524out_err:525 pr_err("QAT: failed to get device out of reset\n");526 return -EFAULT;527}528 529static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,530 unsigned char ae, unsigned int ctx_mask)531{532 unsigned int ctx;533 534 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);535 ctx &= IGNORE_W1C_MASK &536 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));537 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);538}539 540static u64 qat_hal_parity_64bit(u64 word)541{542 word ^= word >> 1;543 word ^= word >> 2;544 word ^= word >> 4;545 word ^= word >> 8;546 word ^= word >> 16;547 word ^= word >> 32;548 return word & 1;549}550 551static u64 qat_hal_set_uword_ecc(u64 uword)552{553 u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,554 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,555 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,556 bit6_mask = 0xdaf69a46910ULL;557 558 /* clear the ecc bits */559 uword &= ~(0x7fULL << 0x2C);560 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;561 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;562 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;563 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;564 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;565 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;566 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;567 return uword;568}569 570void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,571 unsigned char ae, unsigned int uaddr,572 unsigned int words_num, u64 *uword)573{574 unsigned int ustore_addr;575 unsigned int i;576 577 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);578 uaddr |= UA_ECS;579 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);580 for (i = 0; i < words_num; i++) {581 unsigned int uwrd_lo, uwrd_hi;582 u64 tmp;583 584 tmp = qat_hal_set_uword_ecc(uword[i]);585 uwrd_lo = (unsigned int)(tmp & 0xffffffff);586 uwrd_hi = (unsigned int)(tmp >> 0x20);587 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);588 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);589 }590 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);591}592 593static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,594 unsigned char ae, unsigned int ctx_mask)595{596 unsigned int ctx;597 598 ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);599 ctx &= IGNORE_W1C_MASK;600 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;601 ctx |= (ctx_mask << CE_ENABLE_BITPOS);602 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);603}604 605static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)606{607 unsigned long ae_mask = handle->hal_handle->ae_mask;608 unsigned char ae;609 unsigned short reg;610 611 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {612 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {613 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,614 reg, 0);615 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,616 reg, 0);617 }618 }619}620 621static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)622{623 unsigned long ae_mask = handle->hal_handle->ae_mask;624 unsigned char ae;625 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;626 int times = MAX_RETRY_TIMES;627 unsigned int csr_val = 0;628 unsigned int savctx = 0;629 int ret = 0;630 631 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {632 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);633 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);634 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);635 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);636 csr_val &= IGNORE_W1C_MASK;637 if (handle->chip_info->nn)638 csr_val |= CE_NN_MODE;639 640 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);641 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),642 (u64 *)inst);643 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,644 handle->hal_handle->upc_mask &645 INIT_PC_VALUE);646 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);647 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);648 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);649 qat_hal_wr_indr_csr(handle, ae, ctx_mask,650 CTX_SIG_EVENTS_INDIRECT, 0);651 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);652 qat_hal_enable_ctx(handle, ae, ctx_mask);653 }654 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {655 /* wait for AE to finish */656 do {657 ret = qat_hal_wait_cycles(handle, ae, 20, 1);658 } while (ret && times--);659 660 if (times < 0) {661 pr_err("QAT: clear GPR of AE %d failed", ae);662 return -EINVAL;663 }664 qat_hal_disable_ctx(handle, ae, ctx_mask);665 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,666 savctx & ACS_ACNO);667 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,668 INIT_CTX_ENABLE_VALUE);669 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,670 handle->hal_handle->upc_mask &671 INIT_PC_VALUE);672 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);673 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);674 qat_hal_put_wakeup_event(handle, ae, ctx_mask,675 INIT_WAKEUP_EVENTS_VALUE);676 qat_hal_put_sig_event(handle, ae, ctx_mask,677 INIT_SIG_EVENTS_VALUE);678 }679 return 0;680}681 682static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,683 struct adf_accel_dev *accel_dev)684{685 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;686 struct adf_hw_device_data *hw_data = accel_dev->hw_device;687 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);688 unsigned int max_en_ae_id = 0;689 struct adf_bar *sram_bar;690 unsigned int csr_val = 0;691 unsigned long ae_mask;692 unsigned char ae = 0;693 int ret = 0;694 695 handle->pci_dev = pci_info->pci_dev;696 switch (handle->pci_dev->device) {697 case ADF_4XXX_PCI_DEVICE_ID:698 case ADF_401XX_PCI_DEVICE_ID:699 case ADF_402XX_PCI_DEVICE_ID:700 case ADF_420XX_PCI_DEVICE_ID:701 handle->chip_info->mmp_sram_size = 0;702 handle->chip_info->nn = false;703 handle->chip_info->lm2lm3 = true;704 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;705 handle->chip_info->icp_rst_csr = ICP_RESET_CPP0;706 if (handle->pci_dev->device == ADF_420XX_PCI_DEVICE_ID)707 handle->chip_info->icp_rst_mask = 0x100155;708 else709 handle->chip_info->icp_rst_mask = 0x100015;710 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0;711 handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX;712 handle->chip_info->wakeup_event_val = 0x80000000;713 handle->chip_info->fw_auth = true;714 handle->chip_info->css_3k = true;715 handle->chip_info->tgroup_share_ustore = true;716 handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;717 handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;718 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX;719 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX;720 handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;721 handle->chip_info->fcu_loaded_ae_pos = 0;722 723 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX;724 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX;725 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX;726 handle->hal_cap_ae_local_csr_addr_v =727 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v728 + LOCAL_TO_XFER_REG_OFFSET);729 break;730 case PCI_DEVICE_ID_INTEL_QAT_C62X:731 case PCI_DEVICE_ID_INTEL_QAT_C3XXX:732 handle->chip_info->mmp_sram_size = 0;733 handle->chip_info->nn = true;734 handle->chip_info->lm2lm3 = false;735 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;736 handle->chip_info->icp_rst_csr = ICP_RESET;737 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |738 (hw_data->accel_mask << RST_CSR_QAT_LSB);739 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;740 handle->chip_info->misc_ctl_csr = MISC_CONTROL;741 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;742 handle->chip_info->fw_auth = true;743 handle->chip_info->css_3k = false;744 handle->chip_info->tgroup_share_ustore = false;745 handle->chip_info->fcu_ctl_csr = FCU_CONTROL;746 handle->chip_info->fcu_sts_csr = FCU_STATUS;747 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI;748 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;749 handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;750 handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;751 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;752 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;753 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;754 handle->hal_cap_ae_local_csr_addr_v =755 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v756 + LOCAL_TO_XFER_REG_OFFSET);757 break;758 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:759 handle->chip_info->mmp_sram_size = 0x40000;760 handle->chip_info->nn = true;761 handle->chip_info->lm2lm3 = false;762 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;763 handle->chip_info->icp_rst_csr = ICP_RESET;764 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |765 (hw_data->accel_mask << RST_CSR_QAT_LSB);766 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;767 handle->chip_info->misc_ctl_csr = MISC_CONTROL;768 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;769 handle->chip_info->fw_auth = false;770 handle->chip_info->css_3k = false;771 handle->chip_info->tgroup_share_ustore = false;772 handle->chip_info->fcu_ctl_csr = 0;773 handle->chip_info->fcu_sts_csr = 0;774 handle->chip_info->fcu_dram_addr_hi = 0;775 handle->chip_info->fcu_dram_addr_lo = 0;776 handle->chip_info->fcu_loaded_ae_csr = 0;777 handle->chip_info->fcu_loaded_ae_pos = 0;778 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;779 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;780 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;781 handle->hal_cap_ae_local_csr_addr_v =782 (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v783 + LOCAL_TO_XFER_REG_OFFSET);784 break;785 default:786 ret = -EINVAL;787 goto out_err;788 }789 790 if (handle->chip_info->mmp_sram_size > 0) {791 sram_bar =792 &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];793 handle->hal_sram_addr_v = sram_bar->virt_addr;794 }795 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;796 handle->hal_handle->ae_mask = hw_data->ae_mask;797 handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;798 handle->hal_handle->slice_mask = hw_data->accel_mask;799 handle->cfg_ae_mask = ALL_AE_MASK;800 /* create AE objects */801 handle->hal_handle->upc_mask = 0x1ffff;802 handle->hal_handle->max_ustore = 0x4000;803 804 ae_mask = handle->hal_handle->ae_mask;805 for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) {806 handle->hal_handle->aes[ae].free_addr = 0;807 handle->hal_handle->aes[ae].free_size =808 handle->hal_handle->max_ustore;809 handle->hal_handle->aes[ae].ustore_size =810 handle->hal_handle->max_ustore;811 handle->hal_handle->aes[ae].live_ctx_mask =812 ICP_QAT_UCLO_AE_ALL_CTX;813 max_en_ae_id = ae;814 }815 handle->hal_handle->ae_max_num = max_en_ae_id + 1;816 817 /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */818 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {819 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);820 csr_val |= 0x1;821 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);822 }823out_err:824 return ret;825}826 827int qat_hal_init(struct adf_accel_dev *accel_dev)828{829 struct icp_qat_fw_loader_handle *handle;830 int ret = 0;831 832 handle = kzalloc(sizeof(*handle), GFP_KERNEL);833 if (!handle)834 return -ENOMEM;835 836 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);837 if (!handle->hal_handle) {838 ret = -ENOMEM;839 goto out_hal_handle;840 }841 842 handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL);843 if (!handle->chip_info) {844 ret = -ENOMEM;845 goto out_chip_info;846 }847 848 ret = qat_hal_chip_init(handle, accel_dev);849 if (ret) {850 dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n");851 goto out_err;852 }853 854 /* take all AEs out of reset */855 ret = qat_hal_clr_reset(handle);856 if (ret) {857 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");858 goto out_err;859 }860 861 qat_hal_clear_xfer(handle);862 if (!handle->chip_info->fw_auth) {863 ret = qat_hal_clear_gpr(handle);864 if (ret)865 goto out_err;866 }867 868 accel_dev->fw_loader->fw_loader = handle;869 return 0;870 871out_err:872 kfree(handle->chip_info);873out_chip_info:874 kfree(handle->hal_handle);875out_hal_handle:876 kfree(handle);877 return ret;878}879 880void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)881{882 if (!handle)883 return;884 kfree(handle->chip_info);885 kfree(handle->hal_handle);886 kfree(handle);887}888 889int qat_hal_start(struct icp_qat_fw_loader_handle *handle)890{891 unsigned long ae_mask = handle->hal_handle->ae_mask;892 u32 wakeup_val = handle->chip_info->wakeup_event_val;893 u32 fcu_ctl_csr, fcu_sts_csr;894 unsigned int fcu_sts;895 unsigned char ae;896 u32 ae_ctr = 0;897 int retry = 0;898 899 if (handle->chip_info->fw_auth) {900 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;901 fcu_sts_csr = handle->chip_info->fcu_sts_csr;902 ae_ctr = hweight32(ae_mask);903 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);904 do {905 msleep(FW_AUTH_WAIT_PERIOD);906 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);907 if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))908 return ae_ctr;909 } while (retry++ < FW_AUTH_MAX_RETRY);910 pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts);911 return 0;912 } else {913 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {914 qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val);915 qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);916 ae_ctr++;917 }918 return ae_ctr;919 }920}921 922void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,923 unsigned int ctx_mask)924{925 if (!handle->chip_info->fw_auth)926 qat_hal_disable_ctx(handle, ae, ctx_mask);927}928 929void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,930 unsigned char ae, unsigned int ctx_mask, unsigned int upc)931{932 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,933 handle->hal_handle->upc_mask & upc);934}935 936static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,937 unsigned char ae, unsigned int uaddr,938 unsigned int words_num, u64 *uword)939{940 unsigned int i, uwrd_lo, uwrd_hi;941 unsigned int ustore_addr, misc_control;942 943 misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);944 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,945 misc_control & 0xfffffffb);946 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);947 uaddr |= UA_ECS;948 for (i = 0; i < words_num; i++) {949 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);950 uaddr++;951 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);952 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);953 uword[i] = uwrd_hi;954 uword[i] = (uword[i] << 0x20) | uwrd_lo;955 }956 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);957 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);958}959 960void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,961 unsigned char ae, unsigned int uaddr,962 unsigned int words_num, unsigned int *data)963{964 unsigned int i, ustore_addr;965 966 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);967 uaddr |= UA_ECS;968 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);969 for (i = 0; i < words_num; i++) {970 unsigned int uwrd_lo, uwrd_hi, tmp;971 972 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |973 ((data[i] & 0xff00) << 2) |974 (0x3 << 8) | (data[i] & 0xff);975 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);976 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;977 tmp = ((data[i] >> 0x10) & 0xffff);978 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;979 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);980 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);981 }982 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);983}984 985#define MAX_EXEC_INST 100986static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,987 unsigned char ae, unsigned char ctx,988 u64 *micro_inst, unsigned int inst_num,989 int code_off, unsigned int max_cycle,990 unsigned int *endpc)991{992 unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0;993 unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0;994 unsigned int ind_t_index = 0, ind_t_index_byte = 0;995 unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0;996 unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0;997 u64 savuwords[MAX_EXEC_INST];998 unsigned int ind_cnt_sig;999 unsigned int ind_sig, act_sig;1000 unsigned int csr_val = 0, newcsr_val;1001 unsigned int savctx;1002 unsigned int savcc, wakeup_events, savpc;1003 unsigned int ctxarb_ctl, ctx_enables;1004 1005 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {1006 pr_err("QAT: invalid instruction num %d\n", inst_num);1007 return -EINVAL;1008 }1009 /* save current context */1010 ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);1011 ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);1012 ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,1013 INDIRECT_LM_ADDR_0_BYTE_INDEX);1014 ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,1015 INDIRECT_LM_ADDR_1_BYTE_INDEX);1016 if (handle->chip_info->lm2lm3) {1017 ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx,1018 LM_ADDR_2_INDIRECT);1019 ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx,1020 LM_ADDR_3_INDIRECT);1021 ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx,1022 INDIRECT_LM_ADDR_2_BYTE_INDEX);1023 ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx,1024 INDIRECT_LM_ADDR_3_BYTE_INDEX);1025 ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx,1026 INDIRECT_T_INDEX);1027 ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx,1028 INDIRECT_T_INDEX_BYTE_INDEX);1029 }1030 if (inst_num <= MAX_EXEC_INST)1031 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);1032 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);1033 savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);1034 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;1035 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);1036 ctx_enables &= IGNORE_W1C_MASK;1037 savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);1038 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);1039 ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);1040 ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,1041 FUTURE_COUNT_SIGNAL_INDIRECT);1042 ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,1043 CTX_SIG_EVENTS_INDIRECT);1044 act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);1045 /* execute micro codes */1046 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);1047 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);1048 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);1049 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);1050 if (code_off)1051 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);1052 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);1053 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);1054 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);1055 qat_hal_enable_ctx(handle, ae, (1 << ctx));1056 /* wait for micro codes to finish */1057 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)1058 return -EFAULT;1059 if (endpc) {1060 unsigned int ctx_status;1061 1062 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,1063 CTX_STS_INDIRECT);1064 *endpc = ctx_status & handle->hal_handle->upc_mask;1065 }1066 /* retore to saved context */1067 qat_hal_disable_ctx(handle, ae, (1 << ctx));1068 if (inst_num <= MAX_EXEC_INST)1069 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);1070 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);1071 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,1072 handle->hal_handle->upc_mask & savpc);1073 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);1074 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);1075 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);1076 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);1077 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);1078 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);1079 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),1080 LM_ADDR_0_INDIRECT, ind_lm_addr0);1081 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),1082 LM_ADDR_1_INDIRECT, ind_lm_addr1);1083 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),1084 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);1085 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),1086 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);1087 if (handle->chip_info->lm2lm3) {1088 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT,1089 ind_lm_addr2);1090 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT,1091 ind_lm_addr3);1092 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),1093 INDIRECT_LM_ADDR_2_BYTE_INDEX,1094 ind_lm_addr_byte2);1095 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),1096 INDIRECT_LM_ADDR_3_BYTE_INDEX,1097 ind_lm_addr_byte3);1098 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),1099 INDIRECT_T_INDEX, ind_t_index);1100 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),1101 INDIRECT_T_INDEX_BYTE_INDEX,1102 ind_t_index_byte);1103 }1104 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),1105 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);1106 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),1107 CTX_SIG_EVENTS_INDIRECT, ind_sig);1108 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);1109 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);1110 1111 return 0;1112}1113 1114static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,1115 unsigned char ae, unsigned char ctx,1116 enum icp_qat_uof_regtype reg_type,1117 unsigned short reg_num, unsigned int *data)1118{1119 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;1120 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;1121 unsigned short reg_addr;1122 int status = 0;1123 u64 insts, savuword;1124 1125 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);1126 if (reg_addr == BAD_REGADDR) {1127 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);1128 return -EINVAL;1129 }1130 switch (reg_type) {1131 case ICP_GPA_REL:1132 insts = 0xA070000000ull | (reg_addr & 0x3ff);1133 break;1134 default:1135 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);1136 break;1137 }1138 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);1139 ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);1140 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);1141 ctx_enables &= IGNORE_W1C_MASK;1142 if (ctx != (savctx & ACS_ACNO))1143 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,1144 ctx & ACS_ACNO);1145 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);1146 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);1147 ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);1148 uaddr = UA_ECS;1149 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);1150 insts = qat_hal_set_uword_ecc(insts);1151 uwrd_lo = (unsigned int)(insts & 0xffffffff);1152 uwrd_hi = (unsigned int)(insts >> 0x20);1153 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);1154 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);1155 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);1156 /* delay for at least 8 cycles */1157 qat_hal_wait_cycles(handle, ae, 0x8, 0);1158 /*1159 * read ALU output1160 * the instruction should have been executed1161 * prior to clearing the ECS in putUwords1162 */1163 *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);1164 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);1165 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);1166 if (ctx != (savctx & ACS_ACNO))1167 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,1168 savctx & ACS_ACNO);1169 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);1170 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);1171 1172 return status;1173}1174 1175static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,1176 unsigned char ae, unsigned char ctx,1177 enum icp_qat_uof_regtype reg_type,1178 unsigned short reg_num, unsigned int data)1179{1180 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;1181 u64 insts[] = {1182 0x0F440000000ull,1183 0x0F040000000ull,1184 0x0F0000C0300ull,1185 0x0E000010000ull1186 };1187 const int num_inst = ARRAY_SIZE(insts), code_off = 1;1188 const int imm_w1 = 0, imm_w0 = 1;1189 1190 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);1191 if (dest_addr == BAD_REGADDR) {1192 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);1193 return -EINVAL;1194 }1195 1196 data16lo = 0xffff & data;1197 data16hi = 0xffff & (data >> 0x10);1198 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)1199 (0xff & data16hi));1200 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)1201 (0xff & data16lo));1202 switch (reg_type) {1203 case ICP_GPA_REL:1204 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |1205 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);1206 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |1207 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);1208 break;1209 default:1210 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |1211 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);1212 1213 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |1214 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);1215 break;1216 }1217 1218 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,1219 code_off, num_inst * 0x5, NULL);1220}1221 1222int qat_hal_get_ins_num(void)1223{1224 return ARRAY_SIZE(inst_4b);1225}1226 1227static int qat_hal_concat_micro_code(u64 *micro_inst,1228 unsigned int inst_num, unsigned int size,1229 unsigned int addr, unsigned int *value)1230{1231 int i;1232 unsigned int cur_value;1233 const u64 *inst_arr;1234 int fixup_offset;1235 int usize = 0;1236 int orig_num;1237 1238 orig_num = inst_num;1239 cur_value = value[0];1240 inst_arr = inst_4b;1241 usize = ARRAY_SIZE(inst_4b);1242 fixup_offset = inst_num;1243 for (i = 0; i < usize; i++)1244 micro_inst[inst_num++] = inst_arr[i];1245 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));1246 fixup_offset++;1247 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);1248 fixup_offset++;1249 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));1250 fixup_offset++;1251 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));1252 1253 return inst_num - orig_num;1254}1255 1256static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,1257 unsigned char ae, unsigned char ctx,1258 int *pfirst_exec, u64 *micro_inst,1259 unsigned int inst_num)1260{1261 int stat = 0;1262 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;1263 unsigned int gprb0 = 0, gprb1 = 0;1264 1265 if (*pfirst_exec) {1266 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);1267 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);1268 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);1269 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);1270 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);1271 *pfirst_exec = 0;1272 }1273 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,1274 inst_num * 0x5, NULL);1275 if (stat != 0)1276 return -EFAULT;1277 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);1278 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);1279 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);1280 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);1281 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);1282 1283 return 0;1284}1285 1286int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,1287 unsigned char ae,1288 struct icp_qat_uof_batch_init *lm_init_header)1289{1290 struct icp_qat_uof_batch_init *plm_init;1291 u64 *micro_inst_arry;1292 int micro_inst_num;1293 int alloc_inst_size;1294 int first_exec = 1;1295 int stat = 0;1296 1297 plm_init = lm_init_header->next;1298 alloc_inst_size = lm_init_header->size;1299 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)1300 alloc_inst_size = handle->hal_handle->max_ustore;1301 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),1302 GFP_KERNEL);1303 if (!micro_inst_arry)1304 return -ENOMEM;1305 micro_inst_num = 0;1306 while (plm_init) {1307 unsigned int addr, *value, size;1308 1309 ae = plm_init->ae;1310 addr = plm_init->addr;1311 value = plm_init->value;1312 size = plm_init->size;1313 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,1314 micro_inst_num,1315 size, addr, value);1316 plm_init = plm_init->next;1317 }1318 /* exec micro codes */1319 if (micro_inst_arry && micro_inst_num > 0) {1320 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;1321 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,1322 micro_inst_arry,1323 micro_inst_num);1324 }1325 kfree(micro_inst_arry);1326 return stat;1327}1328 1329static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,1330 unsigned char ae, unsigned char ctx,1331 enum icp_qat_uof_regtype reg_type,1332 unsigned short reg_num, unsigned int val)1333{1334 int status = 0;1335 unsigned int reg_addr;1336 unsigned int ctx_enables;1337 unsigned short mask;1338 unsigned short dr_offset = 0x10;1339 1340 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);1341 if (CE_INUSE_CONTEXTS & ctx_enables) {1342 if (ctx & 0x1) {1343 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);1344 return -EINVAL;1345 }1346 mask = 0x1f;1347 dr_offset = 0x20;1348 } else {1349 mask = 0x0f;1350 }1351 if (reg_num & ~mask)1352 return -EINVAL;1353 reg_addr = reg_num + (ctx << 0x5);1354 switch (reg_type) {1355 case ICP_SR_RD_REL:1356 case ICP_SR_REL:1357 SET_AE_XFER(handle, ae, reg_addr, val);1358 break;1359 case ICP_DR_RD_REL:1360 case ICP_DR_REL:1361 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);1362 break;1363 default:1364 status = -EINVAL;1365 break;1366 }1367 return status;1368}1369 1370static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,1371 unsigned char ae, unsigned char ctx,1372 enum icp_qat_uof_regtype reg_type,1373 unsigned short reg_num, unsigned int data)1374{1375 unsigned int gprval, ctx_enables;1376 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,1377 data16low;1378 unsigned short reg_mask;1379 int status = 0;1380 u64 micro_inst[] = {1381 0x0F440000000ull,1382 0x0F040000000ull,1383 0x0A000000000ull,1384 0x0F0000C0300ull,1385 0x0E000010000ull1386 };1387 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;1388 const unsigned short gprnum = 0, dly = num_inst * 0x5;1389 1390 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);1391 if (CE_INUSE_CONTEXTS & ctx_enables) {1392 if (ctx & 0x1) {1393 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);1394 return -EINVAL;1395 }1396 reg_mask = (unsigned short)~0x1f;1397 } else {1398 reg_mask = (unsigned short)~0xf;1399 }1400 if (reg_num & reg_mask)1401 return -EINVAL;1402 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);1403 if (xfr_addr == BAD_REGADDR) {1404 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);1405 return -EINVAL;1406 }1407 status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);1408 if (status) {1409 pr_err("QAT: failed to read register");1410 return status;1411 }1412 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);1413 data16low = 0xffff & data;1414 data16hi = 0xffff & (data >> 0x10);1415 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,1416 (unsigned short)(0xff & data16hi));1417 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,1418 (unsigned short)(0xff & data16low));1419 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |1420 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);1421 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |1422 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);1423 micro_inst[0x2] = micro_inst[0x2] |1424 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);1425 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,1426 code_off, dly, NULL);1427 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);1428 return status;1429}1430 1431static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,1432 unsigned char ae, unsigned char ctx,1433 unsigned short nn, unsigned int val)1434{1435 unsigned int ctx_enables;1436 int stat = 0;1437 1438 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);1439 ctx_enables &= IGNORE_W1C_MASK;1440 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);1441 1442 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);1443 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);1444 return stat;1445}1446 1447static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle1448 *handle, unsigned char ae,1449 unsigned short absreg_num,1450 unsigned short *relreg,1451 unsigned char *ctx)1452{1453 unsigned int ctx_enables;1454 1455 ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);1456 if (ctx_enables & CE_INUSE_CONTEXTS) {1457 /* 4-ctx mode */1458 *relreg = absreg_num & 0x1F;1459 *ctx = (absreg_num >> 0x4) & 0x6;1460 } else {1461 /* 8-ctx mode */1462 *relreg = absreg_num & 0x0F;1463 *ctx = (absreg_num >> 0x4) & 0x7;1464 }1465 return 0;1466}1467 1468int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,1469 unsigned char ae, unsigned long ctx_mask,1470 enum icp_qat_uof_regtype reg_type,1471 unsigned short reg_num, unsigned int regdata)1472{1473 int stat = 0;1474 unsigned short reg;1475 unsigned char ctx = 0;1476 enum icp_qat_uof_regtype type;1477 1478 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)1479 return -EINVAL;1480 1481 do {1482 if (ctx_mask == 0) {1483 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,1484 &ctx);1485 type = reg_type - 1;1486 } else {1487 reg = reg_num;1488 type = reg_type;1489 if (!test_bit(ctx, &ctx_mask))1490 continue;1491 }1492 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);1493 if (stat) {1494 pr_err("QAT: write gpr fail\n");1495 return -EINVAL;1496 }1497 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));1498 1499 return 0;1500}1501 1502int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,1503 unsigned char ae, unsigned long ctx_mask,1504 enum icp_qat_uof_regtype reg_type,1505 unsigned short reg_num, unsigned int regdata)1506{1507 int stat = 0;1508 unsigned short reg;1509 unsigned char ctx = 0;1510 enum icp_qat_uof_regtype type;1511 1512 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)1513 return -EINVAL;1514 1515 do {1516 if (ctx_mask == 0) {1517 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,1518 &ctx);1519 type = reg_type - 3;1520 } else {1521 reg = reg_num;1522 type = reg_type;1523 if (!test_bit(ctx, &ctx_mask))1524 continue;1525 }1526 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,1527 regdata);1528 if (stat) {1529 pr_err("QAT: write wr xfer fail\n");1530 return -EINVAL;1531 }1532 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));1533 1534 return 0;1535}1536 1537int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,1538 unsigned char ae, unsigned long ctx_mask,1539 enum icp_qat_uof_regtype reg_type,1540 unsigned short reg_num, unsigned int regdata)1541{1542 int stat = 0;1543 unsigned short reg;1544 unsigned char ctx = 0;1545 enum icp_qat_uof_regtype type;1546 1547 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)1548 return -EINVAL;1549 1550 do {1551 if (ctx_mask == 0) {1552 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,1553 &ctx);1554 type = reg_type - 3;1555 } else {1556 reg = reg_num;1557 type = reg_type;1558 if (!test_bit(ctx, &ctx_mask))1559 continue;1560 }1561 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,1562 regdata);1563 if (stat) {1564 pr_err("QAT: write rd xfer fail\n");1565 return -EINVAL;1566 }1567 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));1568 1569 return 0;1570}1571 1572int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,1573 unsigned char ae, unsigned long ctx_mask,1574 unsigned short reg_num, unsigned int regdata)1575{1576 int stat = 0;1577 unsigned char ctx;1578 if (!handle->chip_info->nn) {1579 dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n",1580 handle->pci_dev->device);1581 return -EINVAL;1582 }1583 1584 if (ctx_mask == 0)1585 return -EINVAL;1586 1587 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {1588 if (!test_bit(ctx, &ctx_mask))1589 continue;1590 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);1591 if (stat) {1592 pr_err("QAT: write neigh error\n");1593 return -EINVAL;1594 }1595 }1596 1597 return 0;1598}1599