2044 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2017 Marvell4 *5 * Antoine Tenart <antoine.tenart@free-electrons.com>6 */7 8#include <linux/clk.h>9#include <linux/device.h>10#include <linux/dma-mapping.h>11#include <linux/dmapool.h>12#include <linux/firmware.h>13#include <linux/interrupt.h>14#include <linux/module.h>15#include <linux/of_platform.h>16#include <linux/of_irq.h>17#include <linux/pci.h>18#include <linux/platform_device.h>19#include <linux/workqueue.h>20 21#include <crypto/internal/aead.h>22#include <crypto/internal/hash.h>23#include <crypto/internal/skcipher.h>24 25#include "safexcel.h"26 27static u32 max_rings = EIP197_MAX_RINGS;28module_param(max_rings, uint, 0644);29MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");30 31static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)32{33 int i;34 35 /*36 * Map all interfaces/rings to register index 037 * so they can share contexts. Without this, the EIP197 will38 * assume each interface/ring to be in its own memory domain39 * i.e. have its own subset of UNIQUE memory addresses.40 * Which would cause records with the SAME memory address to41 * use DIFFERENT cache buffers, causing both poor cache utilization42 * AND serious coherence/invalidation issues.43 */44 for (i = 0; i < 4; i++)45 writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));46 47 /*48 * Initialize other virtualization regs for cache49 * These may not be in their reset state ...50 */51 for (i = 0; i < priv->config.rings; i++) {52 writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));53 writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));54 writel(EIP197_FLUE_CONFIG_MAGIC,55 priv->base + EIP197_FLUE_CONFIG(i));56 }57 writel(0, priv->base + EIP197_FLUE_OFFSETS);58 writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);59}60 61static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,62 u32 addrmid, int *actbank)63{64 u32 val;65 int curbank;66 67 curbank = addrmid >> 16;68 if (curbank != *actbank) {69 val = readl(priv->base + EIP197_CS_RAM_CTRL);70 val = (val & ~EIP197_CS_BANKSEL_MASK) |71 (curbank << EIP197_CS_BANKSEL_OFS);72 writel(val, priv->base + EIP197_CS_RAM_CTRL);73 *actbank = curbank;74 }75}76 77static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,78 int maxbanks, u32 probemask, u32 stride)79{80 u32 val, addrhi, addrlo, addrmid, addralias, delta, marker;81 int actbank;82 83 /*84 * And probe the actual size of the physically attached cache data RAM85 * Using a binary subdivision algorithm downto 32 byte cache lines.86 */87 addrhi = 1 << (16 + maxbanks);88 addrlo = 0;89 actbank = min(maxbanks - 1, 0);90 while ((addrhi - addrlo) > stride) {91 /* write marker to lowest address in top half */92 addrmid = (addrhi + addrlo) >> 1;93 marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */94 eip197_trc_cache_banksel(priv, addrmid, &actbank);95 writel(marker,96 priv->base + EIP197_CLASSIFICATION_RAMS +97 (addrmid & 0xffff));98 99 /* write invalid markers to possible aliases */100 delta = 1 << __fls(addrmid);101 while (delta >= stride) {102 addralias = addrmid - delta;103 eip197_trc_cache_banksel(priv, addralias, &actbank);104 writel(~marker,105 priv->base + EIP197_CLASSIFICATION_RAMS +106 (addralias & 0xffff));107 delta >>= 1;108 }109 110 /* read back marker from top half */111 eip197_trc_cache_banksel(priv, addrmid, &actbank);112 val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +113 (addrmid & 0xffff));114 115 if ((val & probemask) == marker)116 /* read back correct, continue with top half */117 addrlo = addrmid;118 else119 /* not read back correct, continue with bottom half */120 addrhi = addrmid;121 }122 return addrhi;123}124 125static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,126 int cs_rc_max, int cs_ht_wc)127{128 int i;129 u32 htable_offset, val, offset;130 131 /* Clear all records in administration RAM */132 for (i = 0; i < cs_rc_max; i++) {133 offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;134 135 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |136 EIP197_CS_RC_PREV(EIP197_RC_NULL),137 priv->base + offset);138 139 val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);140 if (i == 0)141 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);142 else if (i == cs_rc_max - 1)143 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);144 writel(val, priv->base + offset + 4);145 /* must also initialize the address key due to ECC! */146 writel(0, priv->base + offset + 8);147 writel(0, priv->base + offset + 12);148 }149 150 /* Clear the hash table entries */151 htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;152 for (i = 0; i < cs_ht_wc; i++)153 writel(GENMASK(29, 0),154 priv->base + EIP197_CLASSIFICATION_RAMS +155 htable_offset + i * sizeof(u32));156}157 158static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv)159{160 u32 val, dsize, asize;161 int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;162 int cs_rc_abs_max, cs_ht_sz;163 int maxbanks;164 165 /* Setup (dummy) virtualization for cache */166 eip197_trc_cache_setupvirt(priv);167 168 /*169 * Enable the record cache memory access and170 * probe the bank select width171 */172 val = readl(priv->base + EIP197_CS_RAM_CTRL);173 val &= ~EIP197_TRC_ENABLE_MASK;174 val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;175 writel(val, priv->base + EIP197_CS_RAM_CTRL);176 val = readl(priv->base + EIP197_CS_RAM_CTRL);177 maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;178 179 /* Clear all ECC errors */180 writel(0, priv->base + EIP197_TRC_ECCCTRL);181 182 /*183 * Make sure the cache memory is accessible by taking record cache into184 * reset. Need data memory access here, not admin access.185 */186 val = readl(priv->base + EIP197_TRC_PARAMS);187 val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;188 writel(val, priv->base + EIP197_TRC_PARAMS);189 190 /* Probed data RAM size in bytes */191 dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32);192 193 /*194 * Now probe the administration RAM size pretty much the same way195 * Except that only the lower 30 bits are writable and we don't need196 * bank selects197 */198 val = readl(priv->base + EIP197_TRC_PARAMS);199 /* admin access now */200 val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);201 writel(val, priv->base + EIP197_TRC_PARAMS);202 203 /* Probed admin RAM size in admin words */204 asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4;205 206 /* Clear any ECC errors detected while probing! */207 writel(0, priv->base + EIP197_TRC_ECCCTRL);208 209 /* Sanity check probing results */210 if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) {211 dev_err(priv->dev, "Record cache probing failed (%d,%d).",212 dsize, asize);213 return -ENODEV;214 }215 216 /*217 * Determine optimal configuration from RAM sizes218 * Note that we assume that the physical RAM configuration is sane219 * Therefore, we don't do any parameter error checking here ...220 */221 222 /* For now, just use a single record format covering everything */223 cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;224 cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;225 226 /*227 * Step #1: How many records will physically fit?228 * Hard upper limit is 1023!229 */230 cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);231 /* Step #2: Need at least 2 words in the admin RAM per record */232 cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));233 /* Step #3: Determine log2 of hash table size */234 cs_ht_sz = __fls(asize - cs_rc_max) - 2;235 /* Step #4: determine current size of hash table in dwords */236 cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */237 /* Step #5: add back excess words and see if we can fit more records */238 cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2));239 240 /* Clear the cache RAMs */241 eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);242 243 /* Disable the record cache memory access */244 val = readl(priv->base + EIP197_CS_RAM_CTRL);245 val &= ~EIP197_TRC_ENABLE_MASK;246 writel(val, priv->base + EIP197_CS_RAM_CTRL);247 248 /* Write head and tail pointers of the record free chain */249 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |250 EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);251 writel(val, priv->base + EIP197_TRC_FREECHAIN);252 253 /* Configure the record cache #1 */254 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |255 EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);256 writel(val, priv->base + EIP197_TRC_PARAMS2);257 258 /* Configure the record cache #2 */259 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |260 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |261 EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);262 writel(val, priv->base + EIP197_TRC_PARAMS);263 264 dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",265 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);266 return 0;267}268 269static void eip197_init_firmware(struct safexcel_crypto_priv *priv)270{271 int pe, i;272 u32 val;273 274 for (pe = 0; pe < priv->config.pes; pe++) {275 /* Configure the token FIFO's */276 writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));277 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));278 279 /* Clear the ICE scratchpad memory */280 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));281 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |282 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |283 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |284 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;285 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));286 287 /* clear the scratchpad RAM using 32 bit writes only */288 for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)289 writel(0, EIP197_PE(priv) +290 EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));291 292 /* Reset the IFPP engine to make its program mem accessible */293 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |294 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |295 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,296 EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));297 298 /* Reset the IPUE engine to make its program mem accessible */299 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |300 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |301 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,302 EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));303 304 /* Enable access to all IFPP program memories */305 writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,306 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));307 308 /* bypass the OCE, if present */309 if (priv->flags & EIP197_OCE)310 writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) +311 EIP197_PE_DEBUG(pe));312 }313 314}315 316static int eip197_write_firmware(struct safexcel_crypto_priv *priv,317 const struct firmware *fw)318{319 u32 val;320 int i;321 322 /* Write the firmware */323 for (i = 0; i < fw->size / sizeof(u32); i++) {324 if (priv->data->fw_little_endian)325 val = le32_to_cpu(((const __le32 *)fw->data)[i]);326 else327 val = be32_to_cpu(((const __be32 *)fw->data)[i]);328 329 writel(val,330 priv->base + EIP197_CLASSIFICATION_RAMS +331 i * sizeof(val));332 }333 334 /* Exclude final 2 NOPs from size */335 return i - EIP197_FW_TERMINAL_NOPS;336}337 338/*339 * If FW is actual production firmware, then poll for its initialization340 * to complete and check if it is good for the HW, otherwise just return OK.341 */342static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)343{344 int pe, pollcnt;345 u32 base, pollofs;346 347 if (fpp)348 pollofs = EIP197_FW_FPP_READY;349 else350 pollofs = EIP197_FW_PUE_READY;351 352 for (pe = 0; pe < priv->config.pes; pe++) {353 base = EIP197_PE_ICE_SCRATCH_RAM(pe);354 pollcnt = EIP197_FW_START_POLLCNT;355 while (pollcnt &&356 (readl_relaxed(EIP197_PE(priv) + base +357 pollofs) != 1)) {358 pollcnt--;359 }360 if (!pollcnt) {361 dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",362 fpp, pe);363 return false;364 }365 }366 return true;367}368 369static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,370 int ipuesz, int ifppsz, int minifw)371{372 int pe;373 u32 val;374 375 for (pe = 0; pe < priv->config.pes; pe++) {376 /* Disable access to all program memory */377 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));378 379 /* Start IFPP microengines */380 if (minifw)381 val = 0;382 else383 val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &384 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |385 EIP197_PE_ICE_UENG_DEBUG_RESET;386 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));387 388 /* Start IPUE microengines */389 if (minifw)390 val = 0;391 else392 val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &393 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |394 EIP197_PE_ICE_UENG_DEBUG_RESET;395 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));396 }397 398 /* For miniFW startup, there is no initialization, so always succeed */399 if (minifw)400 return true;401 402 /* Wait until all the firmwares have properly started up */403 if (!poll_fw_ready(priv, 1))404 return false;405 if (!poll_fw_ready(priv, 0))406 return false;407 408 return true;409}410 411static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)412{413 const char *fw_name[] = {"ifpp.bin", "ipue.bin"};414 const struct firmware *fw[FW_NB];415 char fw_path[37], *dir = NULL;416 int i, j, ret = 0, pe;417 int ipuesz, ifppsz, minifw = 0;418 419 if (priv->data->version == EIP197D_MRVL)420 dir = "eip197d";421 else if (priv->data->version == EIP197B_MRVL ||422 priv->data->version == EIP197_DEVBRD)423 dir = "eip197b";424 else if (priv->data->version == EIP197C_MXL)425 dir = "eip197c";426 else427 return -ENODEV;428 429retry_fw:430 for (i = 0; i < FW_NB; i++) {431 snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);432 ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);433 if (ret) {434 if (minifw || priv->data->version != EIP197B_MRVL)435 goto release_fw;436 437 /* Fallback to the old firmware location for the438 * EIP197b.439 */440 ret = firmware_request_nowarn(&fw[i], fw_name[i],441 priv->dev);442 if (ret)443 goto release_fw;444 }445 }446 447 eip197_init_firmware(priv);448 449 ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);450 451 /* Enable access to IPUE program memories */452 for (pe = 0; pe < priv->config.pes; pe++)453 writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,454 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));455 456 ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);457 458 if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {459 dev_dbg(priv->dev, "Firmware loaded successfully\n");460 return 0;461 }462 463 ret = -ENODEV;464 465release_fw:466 for (j = 0; j < i; j++)467 release_firmware(fw[j]);468 469 if (!minifw) {470 /* Retry with minifw path */471 dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");472 dir = "eip197_minifw";473 minifw = 1;474 goto retry_fw;475 }476 477 dev_err(priv->dev, "Firmware load failed.\n");478 479 return ret;480}481 482static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)483{484 u32 cd_size_rnd, val;485 int i, cd_fetch_cnt;486 487 cd_size_rnd = (priv->config.cd_size +488 (BIT(priv->hwconfig.hwdataw) - 1)) >>489 priv->hwconfig.hwdataw;490 /* determine number of CD's we can fetch into the CD FIFO as 1 block */491 if (priv->flags & SAFEXCEL_HW_EIP197) {492 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */493 cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;494 cd_fetch_cnt = min_t(uint, cd_fetch_cnt,495 (priv->config.pes * EIP197_FETCH_DEPTH));496 } else {497 /* for the EIP97, just fetch all that fits minus 1 */498 cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /499 cd_size_rnd) - 1;500 }501 /*502 * Since we're using command desc's way larger than formally specified,503 * we need to check whether we can fit even 1 for low-end EIP196's!504 */505 if (!cd_fetch_cnt) {506 dev_err(priv->dev, "Unable to fit even 1 command desc!\n");507 return -ENODEV;508 }509 510 for (i = 0; i < priv->config.rings; i++) {511 /* ring base address */512 writel(lower_32_bits(priv->ring[i].cdr.base_dma),513 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);514 writel(upper_32_bits(priv->ring[i].cdr.base_dma),515 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);516 517 writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP |518 (priv->config.cd_offset << 14) | priv->config.cd_size,519 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);520 writel(((cd_fetch_cnt *521 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |522 (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))),523 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);524 525 /* Configure DMA tx control */526 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);527 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);528 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);529 530 /* clear any pending interrupt */531 writel(GENMASK(5, 0),532 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);533 }534 535 return 0;536}537 538static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)539{540 u32 rd_size_rnd, val;541 int i, rd_fetch_cnt;542 543 /* determine number of RD's we can fetch into the FIFO as one block */544 rd_size_rnd = (EIP197_RD64_FETCH_SIZE +545 (BIT(priv->hwconfig.hwdataw) - 1)) >>546 priv->hwconfig.hwdataw;547 if (priv->flags & SAFEXCEL_HW_EIP197) {548 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */549 rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;550 rd_fetch_cnt = min_t(uint, rd_fetch_cnt,551 (priv->config.pes * EIP197_FETCH_DEPTH));552 } else {553 /* for the EIP97, just fetch all that fits minus 1 */554 rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /555 rd_size_rnd) - 1;556 }557 558 for (i = 0; i < priv->config.rings; i++) {559 /* ring base address */560 writel(lower_32_bits(priv->ring[i].rdr.base_dma),561 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);562 writel(upper_32_bits(priv->ring[i].rdr.base_dma),563 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);564 565 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) |566 priv->config.rd_size,567 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);568 569 writel(((rd_fetch_cnt *570 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |571 (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))),572 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);573 574 /* Configure DMA tx control */575 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);576 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);577 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;578 writel(val,579 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);580 581 /* clear any pending interrupt */582 writel(GENMASK(7, 0),583 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);584 585 /* enable ring interrupt */586 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));587 val |= EIP197_RDR_IRQ(i);588 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));589 }590 591 return 0;592}593 594static int safexcel_hw_init(struct safexcel_crypto_priv *priv)595{596 u32 val;597 int i, ret, pe, opbuflo, opbufhi;598 599 dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",600 priv->config.pes, priv->config.rings);601 602 /*603 * For EIP197's only set maximum number of TX commands to 2^5 = 32604 * Skip for the EIP97 as it does not have this field.605 */606 if (priv->flags & SAFEXCEL_HW_EIP197) {607 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);608 val |= EIP197_MST_CTRL_TX_MAX_CMD(5);609 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);610 }611 612 /* Configure wr/rd cache values */613 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |614 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),615 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);616 617 /* Interrupts reset */618 619 /* Disable all global interrupts */620 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);621 622 /* Clear any pending interrupt */623 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);624 625 /* Processing Engine configuration */626 for (pe = 0; pe < priv->config.pes; pe++) {627 /* Data Fetch Engine configuration */628 629 /* Reset all DFE threads */630 writel(EIP197_DxE_THR_CTRL_RESET_PE,631 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));632 633 if (priv->flags & EIP197_PE_ARB)634 /* Reset HIA input interface arbiter (if present) */635 writel(EIP197_HIA_RA_PE_CTRL_RESET,636 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));637 638 /* DMA transfer size to use */639 val = EIP197_HIA_DFE_CFG_DIS_DEBUG;640 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |641 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);642 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |643 EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);644 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);645 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);646 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));647 648 /* Leave the DFE threads reset state */649 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));650 651 /* Configure the processing engine thresholds */652 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |653 EIP197_PE_IN_xBUF_THRES_MAX(9),654 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));655 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |656 EIP197_PE_IN_xBUF_THRES_MAX(7),657 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));658 659 if (priv->flags & SAFEXCEL_HW_EIP197)660 /* enable HIA input interface arbiter and rings */661 writel(EIP197_HIA_RA_PE_CTRL_EN |662 GENMASK(priv->config.rings - 1, 0),663 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));664 665 /* Data Store Engine configuration */666 667 /* Reset all DSE threads */668 writel(EIP197_DxE_THR_CTRL_RESET_PE,669 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));670 671 /* Wait for all DSE threads to complete */672 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &673 GENMASK(15, 12)) != GENMASK(15, 12))674 ;675 676 /* DMA transfer size to use */677 if (priv->hwconfig.hwnumpes > 4) {678 opbuflo = 9;679 opbufhi = 10;680 } else {681 opbuflo = 7;682 opbufhi = 8;683 }684 val = EIP197_HIA_DSE_CFG_DIS_DEBUG;685 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) |686 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi);687 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);688 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;689 /* FIXME: instability issues can occur for EIP97 but disabling690 * it impacts performance.691 */692 if (priv->flags & SAFEXCEL_HW_EIP197)693 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;694 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));695 696 /* Leave the DSE threads reset state */697 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));698 699 /* Configure the processing engine thresholds */700 writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) |701 EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi),702 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));703 704 /* Processing Engine configuration */705 706 /* Token & context configuration */707 val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |708 EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |709 EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;710 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));711 712 /* H/W capabilities selection: just enable everything */713 writel(EIP197_FUNCTION_ALL,714 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));715 writel(EIP197_FUNCTION_ALL,716 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));717 }718 719 /* Command Descriptor Rings prepare */720 for (i = 0; i < priv->config.rings; i++) {721 /* Clear interrupts for this ring */722 writel(GENMASK(31, 0),723 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));724 725 /* Disable external triggering */726 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);727 728 /* Clear the pending prepared counter */729 writel(EIP197_xDR_PREP_CLR_COUNT,730 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);731 732 /* Clear the pending processed counter */733 writel(EIP197_xDR_PROC_CLR_COUNT,734 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);735 736 writel(0,737 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);738 writel(0,739 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);740 741 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset),742 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);743 }744 745 /* Result Descriptor Ring prepare */746 for (i = 0; i < priv->config.rings; i++) {747 /* Disable external triggering*/748 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);749 750 /* Clear the pending prepared counter */751 writel(EIP197_xDR_PREP_CLR_COUNT,752 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);753 754 /* Clear the pending processed counter */755 writel(EIP197_xDR_PROC_CLR_COUNT,756 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);757 758 writel(0,759 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);760 writel(0,761 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);762 763 /* Ring size */764 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset),765 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);766 }767 768 for (pe = 0; pe < priv->config.pes; pe++) {769 /* Enable command descriptor rings */770 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),771 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));772 773 /* Enable result descriptor rings */774 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),775 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));776 }777 778 /* Clear any HIA interrupt */779 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);780 781 if (priv->flags & EIP197_SIMPLE_TRC) {782 writel(EIP197_STRC_CONFIG_INIT |783 EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) |784 EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC),785 priv->base + EIP197_STRC_CONFIG);786 writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE,787 EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0));788 } else if (priv->flags & SAFEXCEL_HW_EIP197) {789 ret = eip197_trc_cache_init(priv);790 if (ret)791 return ret;792 }793 794 if (priv->flags & EIP197_ICE) {795 ret = eip197_load_firmwares(priv);796 if (ret)797 return ret;798 }799 800 return safexcel_hw_setup_cdesc_rings(priv) ?:801 safexcel_hw_setup_rdesc_rings(priv) ?:802 0;803}804 805/* Called with ring's lock taken */806static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,807 int ring)808{809 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);810 811 if (!coal)812 return;813 814 /* Configure when we want an interrupt */815 writel(EIP197_HIA_RDR_THRESH_PKT_MODE |816 EIP197_HIA_RDR_THRESH_PROC_PKT(coal),817 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);818}819 820void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)821{822 struct crypto_async_request *req, *backlog;823 struct safexcel_context *ctx;824 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;825 826 /* If a request wasn't properly dequeued because of a lack of resources,827 * proceeded it first,828 */829 req = priv->ring[ring].req;830 backlog = priv->ring[ring].backlog;831 if (req)832 goto handle_req;833 834 while (true) {835 spin_lock_bh(&priv->ring[ring].queue_lock);836 backlog = crypto_get_backlog(&priv->ring[ring].queue);837 req = crypto_dequeue_request(&priv->ring[ring].queue);838 spin_unlock_bh(&priv->ring[ring].queue_lock);839 840 if (!req) {841 priv->ring[ring].req = NULL;842 priv->ring[ring].backlog = NULL;843 goto finalize;844 }845 846handle_req:847 ctx = crypto_tfm_ctx(req->tfm);848 ret = ctx->send(req, ring, &commands, &results);849 if (ret)850 goto request_failed;851 852 if (backlog)853 crypto_request_complete(backlog, -EINPROGRESS);854 855 /* In case the send() helper did not issue any command to push856 * to the engine because the input data was cached, continue to857 * dequeue other requests as this is valid and not an error.858 */859 if (!commands && !results)860 continue;861 862 cdesc += commands;863 rdesc += results;864 nreq++;865 }866 867request_failed:868 /* Not enough resources to handle all the requests. Bail out and save869 * the request and the backlog for the next dequeue call (per-ring).870 */871 priv->ring[ring].req = req;872 priv->ring[ring].backlog = backlog;873 874finalize:875 if (!nreq)876 return;877 878 spin_lock_bh(&priv->ring[ring].lock);879 880 priv->ring[ring].requests += nreq;881 882 if (!priv->ring[ring].busy) {883 safexcel_try_push_requests(priv, ring);884 priv->ring[ring].busy = true;885 }886 887 spin_unlock_bh(&priv->ring[ring].lock);888 889 /* let the RDR know we have pending descriptors */890 writel((rdesc * priv->config.rd_offset),891 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);892 893 /* let the CDR know we have pending descriptors */894 writel((cdesc * priv->config.cd_offset),895 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);896}897 898inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,899 void *rdp)900{901 struct safexcel_result_desc *rdesc = rdp;902 struct result_data_desc *result_data = rdp + priv->config.res_offset;903 904 if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */905 ((!rdesc->descriptor_overflow) &&906 (!rdesc->buffer_overflow) &&907 (!result_data->error_code))))908 return 0;909 910 if (rdesc->descriptor_overflow)911 dev_err(priv->dev, "Descriptor overflow detected");912 913 if (rdesc->buffer_overflow)914 dev_err(priv->dev, "Buffer overflow detected");915 916 if (result_data->error_code & 0x4066) {917 /* Fatal error (bits 1,2,5,6 & 14) */918 dev_err(priv->dev,919 "result descriptor error (%x)",920 result_data->error_code);921 922 return -EIO;923 } else if (result_data->error_code &924 (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {925 /*926 * Give priority over authentication fails:927 * Blocksize, length & overflow errors,928 * something wrong with the input!929 */930 return -EINVAL;931 } else if (result_data->error_code & BIT(9)) {932 /* Authentication failed */933 return -EBADMSG;934 }935 936 /* All other non-fatal errors */937 return -EINVAL;938}939 940inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,941 int ring,942 struct safexcel_result_desc *rdesc,943 struct crypto_async_request *req)944{945 int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);946 947 priv->ring[ring].rdr_req[i] = req;948}949 950inline struct crypto_async_request *951safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)952{953 int i = safexcel_ring_first_rdr_index(priv, ring);954 955 return priv->ring[ring].rdr_req[i];956}957 958void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)959{960 struct safexcel_command_desc *cdesc;961 962 /* Acknowledge the command descriptors */963 do {964 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);965 if (IS_ERR(cdesc)) {966 dev_err(priv->dev,967 "Could not retrieve the command descriptor\n");968 return;969 }970 } while (!cdesc->last_seg);971}972 973int safexcel_invalidate_cache(struct crypto_async_request *async,974 struct safexcel_crypto_priv *priv,975 dma_addr_t ctxr_dma, int ring)976{977 struct safexcel_command_desc *cdesc;978 struct safexcel_result_desc *rdesc;979 struct safexcel_token *dmmy;980 int ret = 0;981 982 /* Prepare command descriptor */983 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma,984 &dmmy);985 if (IS_ERR(cdesc))986 return PTR_ERR(cdesc);987 988 cdesc->control_data.type = EIP197_TYPE_EXTENDED;989 cdesc->control_data.options = 0;990 cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK;991 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;992 993 /* Prepare result descriptor */994 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);995 996 if (IS_ERR(rdesc)) {997 ret = PTR_ERR(rdesc);998 goto cdesc_rollback;999 }1000 1001 safexcel_rdr_req_set(priv, ring, rdesc, async);1002 1003 return ret;1004 1005cdesc_rollback:1006 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);1007 1008 return ret;1009}1010 1011static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,1012 int ring)1013{1014 struct crypto_async_request *req;1015 struct safexcel_context *ctx;1016 int ret, i, nreq, ndesc, tot_descs, handled = 0;1017 bool should_complete;1018 1019handle_results:1020 tot_descs = 0;1021 1022 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);1023 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;1024 nreq &= EIP197_xDR_PROC_xD_PKT_MASK;1025 if (!nreq)1026 goto requests_left;1027 1028 for (i = 0; i < nreq; i++) {1029 req = safexcel_rdr_req_get(priv, ring);1030 1031 ctx = crypto_tfm_ctx(req->tfm);1032 ndesc = ctx->handle_result(priv, ring, req,1033 &should_complete, &ret);1034 if (ndesc < 0) {1035 dev_err(priv->dev, "failed to handle result (%d)\n",1036 ndesc);1037 goto acknowledge;1038 }1039 1040 if (should_complete) {1041 local_bh_disable();1042 crypto_request_complete(req, ret);1043 local_bh_enable();1044 }1045 1046 tot_descs += ndesc;1047 handled++;1048 }1049 1050acknowledge:1051 if (i)1052 writel(EIP197_xDR_PROC_xD_PKT(i) |1053 (tot_descs * priv->config.rd_offset),1054 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);1055 1056 /* If the number of requests overflowed the counter, try to proceed more1057 * requests.1058 */1059 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)1060 goto handle_results;1061 1062requests_left:1063 spin_lock_bh(&priv->ring[ring].lock);1064 1065 priv->ring[ring].requests -= handled;1066 safexcel_try_push_requests(priv, ring);1067 1068 if (!priv->ring[ring].requests)1069 priv->ring[ring].busy = false;1070 1071 spin_unlock_bh(&priv->ring[ring].lock);1072}1073 1074static void safexcel_dequeue_work(struct work_struct *work)1075{1076 struct safexcel_work_data *data =1077 container_of(work, struct safexcel_work_data, work);1078 1079 safexcel_dequeue(data->priv, data->ring);1080}1081 1082struct safexcel_ring_irq_data {1083 struct safexcel_crypto_priv *priv;1084 int ring;1085};1086 1087static irqreturn_t safexcel_irq_ring(int irq, void *data)1088{1089 struct safexcel_ring_irq_data *irq_data = data;1090 struct safexcel_crypto_priv *priv = irq_data->priv;1091 int ring = irq_data->ring, rc = IRQ_NONE;1092 u32 status, stat;1093 1094 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));1095 if (!status)1096 return rc;1097 1098 /* RDR interrupts */1099 if (status & EIP197_RDR_IRQ(ring)) {1100 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);1101 1102 if (unlikely(stat & EIP197_xDR_ERR)) {1103 /*1104 * Fatal error, the RDR is unusable and must be1105 * reinitialized. This should not happen under1106 * normal circumstances.1107 */1108 dev_err(priv->dev, "RDR: fatal error.\n");1109 } else if (likely(stat & EIP197_xDR_THRESH)) {1110 rc = IRQ_WAKE_THREAD;1111 }1112 1113 /* ACK the interrupts */1114 writel(stat & 0xff,1115 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);1116 }1117 1118 /* ACK the interrupts */1119 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));1120 1121 return rc;1122}1123 1124static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)1125{1126 struct safexcel_ring_irq_data *irq_data = data;1127 struct safexcel_crypto_priv *priv = irq_data->priv;1128 int ring = irq_data->ring;1129 1130 safexcel_handle_result_descriptor(priv, ring);1131 1132 queue_work(priv->ring[ring].workqueue,1133 &priv->ring[ring].work_data.work);1134 1135 return IRQ_HANDLED;1136}1137 1138static int safexcel_request_ring_irq(void *pdev, int irqid,1139 int is_pci_dev,1140 int ring_id,1141 irq_handler_t handler,1142 irq_handler_t threaded_handler,1143 struct safexcel_ring_irq_data *ring_irq_priv)1144{1145 int ret, irq, cpu;1146 struct device *dev;1147 1148 if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {1149 struct pci_dev *pci_pdev = pdev;1150 1151 dev = &pci_pdev->dev;1152 irq = pci_irq_vector(pci_pdev, irqid);1153 if (irq < 0) {1154 dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",1155 irqid, irq);1156 return irq;1157 }1158 } else if (IS_ENABLED(CONFIG_OF)) {1159 struct platform_device *plf_pdev = pdev;1160 char irq_name[6] = {0}; /* "ringX\0" */1161 1162 snprintf(irq_name, 6, "ring%d", irqid);1163 dev = &plf_pdev->dev;1164 irq = platform_get_irq_byname(plf_pdev, irq_name);1165 1166 if (irq < 0)1167 return irq;1168 } else {1169 return -ENXIO;1170 }1171 1172 ret = devm_request_threaded_irq(dev, irq, handler,1173 threaded_handler, IRQF_ONESHOT,1174 dev_name(dev), ring_irq_priv);1175 if (ret) {1176 dev_err(dev, "unable to request IRQ %d\n", irq);1177 return ret;1178 }1179 1180 /* Set affinity */1181 cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE);1182 irq_set_affinity_hint(irq, get_cpu_mask(cpu));1183 1184 return irq;1185}1186 1187static struct safexcel_alg_template *safexcel_algs[] = {1188 &safexcel_alg_ecb_des,1189 &safexcel_alg_cbc_des,1190 &safexcel_alg_ecb_des3_ede,1191 &safexcel_alg_cbc_des3_ede,1192 &safexcel_alg_ecb_aes,1193 &safexcel_alg_cbc_aes,1194 &safexcel_alg_ctr_aes,1195 &safexcel_alg_md5,1196 &safexcel_alg_sha1,1197 &safexcel_alg_sha224,1198 &safexcel_alg_sha256,1199 &safexcel_alg_sha384,1200 &safexcel_alg_sha512,1201 &safexcel_alg_hmac_md5,1202 &safexcel_alg_hmac_sha1,1203 &safexcel_alg_hmac_sha224,1204 &safexcel_alg_hmac_sha256,1205 &safexcel_alg_hmac_sha384,1206 &safexcel_alg_hmac_sha512,1207 &safexcel_alg_authenc_hmac_sha1_cbc_aes,1208 &safexcel_alg_authenc_hmac_sha224_cbc_aes,1209 &safexcel_alg_authenc_hmac_sha256_cbc_aes,1210 &safexcel_alg_authenc_hmac_sha384_cbc_aes,1211 &safexcel_alg_authenc_hmac_sha512_cbc_aes,1212 &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,1213 &safexcel_alg_authenc_hmac_sha1_ctr_aes,1214 &safexcel_alg_authenc_hmac_sha224_ctr_aes,1215 &safexcel_alg_authenc_hmac_sha256_ctr_aes,1216 &safexcel_alg_authenc_hmac_sha384_ctr_aes,1217 &safexcel_alg_authenc_hmac_sha512_ctr_aes,1218 &safexcel_alg_xts_aes,1219 &safexcel_alg_gcm,1220 &safexcel_alg_ccm,1221 &safexcel_alg_crc32,1222 &safexcel_alg_cbcmac,1223 &safexcel_alg_xcbcmac,1224 &safexcel_alg_cmac,1225 &safexcel_alg_chacha20,1226 &safexcel_alg_chachapoly,1227 &safexcel_alg_chachapoly_esp,1228 &safexcel_alg_sm3,1229 &safexcel_alg_hmac_sm3,1230 &safexcel_alg_ecb_sm4,1231 &safexcel_alg_cbc_sm4,1232 &safexcel_alg_ctr_sm4,1233 &safexcel_alg_authenc_hmac_sha1_cbc_sm4,1234 &safexcel_alg_authenc_hmac_sm3_cbc_sm4,1235 &safexcel_alg_authenc_hmac_sha1_ctr_sm4,1236 &safexcel_alg_authenc_hmac_sm3_ctr_sm4,1237 &safexcel_alg_sha3_224,1238 &safexcel_alg_sha3_256,1239 &safexcel_alg_sha3_384,1240 &safexcel_alg_sha3_512,1241 &safexcel_alg_hmac_sha3_224,1242 &safexcel_alg_hmac_sha3_256,1243 &safexcel_alg_hmac_sha3_384,1244 &safexcel_alg_hmac_sha3_512,1245 &safexcel_alg_authenc_hmac_sha1_cbc_des,1246 &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,1247 &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,1248 &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,1249 &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,1250 &safexcel_alg_authenc_hmac_sha256_cbc_des,1251 &safexcel_alg_authenc_hmac_sha224_cbc_des,1252 &safexcel_alg_authenc_hmac_sha512_cbc_des,1253 &safexcel_alg_authenc_hmac_sha384_cbc_des,1254 &safexcel_alg_rfc4106_gcm,1255 &safexcel_alg_rfc4543_gcm,1256 &safexcel_alg_rfc4309_ccm,1257};1258 1259static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)1260{1261 int i, j, ret = 0;1262 1263 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {1264 safexcel_algs[i]->priv = priv;1265 1266 /* Do we have all required base algorithms available? */1267 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=1268 safexcel_algs[i]->algo_mask)1269 /* No, so don't register this ciphersuite */1270 continue;1271 1272 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)1273 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);1274 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)1275 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);1276 else1277 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);1278 1279 if (ret)1280 goto fail;1281 }1282 1283 return 0;1284 1285fail:1286 for (j = 0; j < i; j++) {1287 /* Do we have all required base algorithms available? */1288 if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=1289 safexcel_algs[j]->algo_mask)1290 /* No, so don't unregister this ciphersuite */1291 continue;1292 1293 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)1294 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);1295 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)1296 crypto_unregister_aead(&safexcel_algs[j]->alg.aead);1297 else1298 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);1299 }1300 1301 return ret;1302}1303 1304static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)1305{1306 int i;1307 1308 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {1309 /* Do we have all required base algorithms available? */1310 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=1311 safexcel_algs[i]->algo_mask)1312 /* No, so don't unregister this ciphersuite */1313 continue;1314 1315 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)1316 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);1317 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)1318 crypto_unregister_aead(&safexcel_algs[i]->alg.aead);1319 else1320 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);1321 }1322}1323 1324static void safexcel_configure(struct safexcel_crypto_priv *priv)1325{1326 u32 mask = BIT(priv->hwconfig.hwdataw) - 1;1327 1328 priv->config.pes = priv->hwconfig.hwnumpes;1329 priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings);1330 /* Cannot currently support more rings than we have ring AICs! */1331 priv->config.rings = min_t(u32, priv->config.rings,1332 priv->hwconfig.hwnumraic);1333 1334 priv->config.cd_size = EIP197_CD64_FETCH_SIZE;1335 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;1336 priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask;1337 1338 /* res token is behind the descr, but ofs must be rounded to buswdth */1339 priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask;1340 /* now the size of the descr is this 1st part plus the result struct */1341 priv->config.rd_size = priv->config.res_offset +1342 EIP197_RD64_RESULT_SIZE;1343 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;1344 1345 /* convert dwords to bytes */1346 priv->config.cd_offset *= sizeof(u32);1347 priv->config.cdsh_offset *= sizeof(u32);1348 priv->config.rd_offset *= sizeof(u32);1349 priv->config.res_offset *= sizeof(u32);1350}1351 1352static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)1353{1354 struct safexcel_register_offsets *offsets = &priv->offsets;1355 1356 if (priv->flags & SAFEXCEL_HW_EIP197) {1357 offsets->hia_aic = EIP197_HIA_AIC_BASE;1358 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;1359 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;1360 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE;1361 offsets->hia_dfe = EIP197_HIA_DFE_BASE;1362 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE;1363 offsets->hia_dse = EIP197_HIA_DSE_BASE;1364 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;1365 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;1366 offsets->pe = EIP197_PE_BASE;1367 offsets->global = EIP197_GLOBAL_BASE;1368 } else {1369 offsets->hia_aic = EIP97_HIA_AIC_BASE;1370 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;1371 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;1372 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE;1373 offsets->hia_dfe = EIP97_HIA_DFE_BASE;1374 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE;1375 offsets->hia_dse = EIP97_HIA_DSE_BASE;1376 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;1377 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;1378 offsets->pe = EIP97_PE_BASE;1379 offsets->global = EIP97_GLOBAL_BASE;1380 }1381}1382 1383/*1384 * Generic part of probe routine, shared by platform and PCI driver1385 *1386 * Assumes IO resources have been mapped, private data mem has been allocated,1387 * clocks have been enabled, device pointer has been assigned etc.1388 *1389 */1390static int safexcel_probe_generic(void *pdev,1391 struct safexcel_crypto_priv *priv,1392 int is_pci_dev)1393{1394 struct device *dev = priv->dev;1395 u32 peid, version, mask, val, hiaopt, hwopt, peopt;1396 int i, ret, hwctg;1397 1398 priv->context_pool = dmam_pool_create("safexcel-context", dev,1399 sizeof(struct safexcel_context_record),1400 1, 0);1401 if (!priv->context_pool)1402 return -ENOMEM;1403 1404 /*1405 * First try the EIP97 HIA version regs1406 * For the EIP197, this is guaranteed to NOT return any of the test1407 * values1408 */1409 version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);1410 1411 mask = 0; /* do not swap */1412 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {1413 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);1414 } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {1415 /* read back byte-swapped, so complement byte swap bits */1416 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;1417 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);1418 } else {1419 /* So it wasn't an EIP97 ... maybe it's an EIP197? */1420 version = readl(priv->base + EIP197_HIA_AIC_BASE +1421 EIP197_HIA_VERSION);1422 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {1423 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);1424 priv->flags |= SAFEXCEL_HW_EIP197;1425 } else if (EIP197_REG_HI16(version) ==1426 EIP197_HIA_VERSION_BE) {1427 /* read back byte-swapped, so complement swap bits */1428 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;1429 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);1430 priv->flags |= SAFEXCEL_HW_EIP197;1431 } else {1432 return -ENODEV;1433 }1434 }1435 1436 /* Now initialize the reg offsets based on the probing info so far */1437 safexcel_init_register_offsets(priv);1438 1439 /*1440 * If the version was read byte-swapped, we need to flip the device1441 * swapping Keep in mind here, though, that what we write will also be1442 * byte-swapped ...1443 */1444 if (mask) {1445 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);1446 val = val ^ (mask >> 24); /* toggle byte swap bits */1447 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);1448 }1449 1450 /*1451 * We're not done probing yet! We may fall through to here if no HIA1452 * was found at all. So, with the endianness presumably correct now and1453 * the offsets setup, *really* probe for the EIP97/EIP197.1454 */1455 version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);1456 if (((priv->flags & SAFEXCEL_HW_EIP197) &&1457 (EIP197_REG_LO16(version) != EIP197_VERSION_LE) &&1458 (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) ||1459 ((!(priv->flags & SAFEXCEL_HW_EIP197) &&1460 (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {1461 /*1462 * We did not find the device that matched our initial probing1463 * (or our initial probing failed) Report appropriate error.1464 */1465 dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n",1466 version);1467 return -ENODEV;1468 }1469 1470 priv->hwconfig.hwver = EIP197_VERSION_MASK(version);1471 hwctg = version >> 28;1472 peid = version & 255;1473 1474 /* Detect EIP206 processing pipe */1475 version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0));1476 if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) {1477 dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid);1478 return -ENODEV;1479 }1480 priv->hwconfig.ppver = EIP197_VERSION_MASK(version);1481 1482 /* Detect EIP96 packet engine and version */1483 version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));1484 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {1485 dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);1486 return -ENODEV;1487 }1488 priv->hwconfig.pever = EIP197_VERSION_MASK(version);1489 1490 hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS);1491 hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);1492 1493 priv->hwconfig.icever = 0;1494 priv->hwconfig.ocever = 0;1495 priv->hwconfig.psever = 0;1496 if (priv->flags & SAFEXCEL_HW_EIP197) {1497 /* EIP197 */1498 peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0));1499 1500 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &1501 EIP197_HWDATAW_MASK;1502 priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &1503 EIP197_CFSIZE_MASK) +1504 EIP197_CFSIZE_ADJUST;1505 priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &1506 EIP197_RFSIZE_MASK) +1507 EIP197_RFSIZE_ADJUST;1508 priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) &1509 EIP197_N_PES_MASK;1510 priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &1511 EIP197_N_RINGS_MASK;1512 if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB)1513 priv->flags |= EIP197_PE_ARB;1514 if (EIP206_OPT_ICE_TYPE(peopt) == 1) {1515 priv->flags |= EIP197_ICE;1516 /* Detect ICE EIP207 class. engine and version */1517 version = readl(EIP197_PE(priv) +1518 EIP197_PE_ICE_VERSION(0));1519 if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {1520 dev_err(dev, "EIP%d: ICE EIP207 not detected.\n",1521 peid);1522 return -ENODEV;1523 }1524 priv->hwconfig.icever = EIP197_VERSION_MASK(version);1525 }1526 if (EIP206_OPT_OCE_TYPE(peopt) == 1) {1527 priv->flags |= EIP197_OCE;1528 /* Detect EIP96PP packet stream editor and version */1529 version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0));1530 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {1531 dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid);1532 return -ENODEV;1533 }1534 priv->hwconfig.psever = EIP197_VERSION_MASK(version);1535 /* Detect OCE EIP207 class. engine and version */1536 version = readl(EIP197_PE(priv) +1537 EIP197_PE_ICE_VERSION(0));1538 if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {1539 dev_err(dev, "EIP%d: OCE EIP207 not detected.\n",1540 peid);1541 return -ENODEV;1542 }1543 priv->hwconfig.ocever = EIP197_VERSION_MASK(version);1544 }1545 /* If not a full TRC, then assume simple TRC */1546 if (!(hwopt & EIP197_OPT_HAS_TRC))1547 priv->flags |= EIP197_SIMPLE_TRC;1548 /* EIP197 always has SOME form of TRC */1549 priv->flags |= EIP197_TRC_CACHE;1550 } else {1551 /* EIP97 */1552 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &1553 EIP97_HWDATAW_MASK;1554 priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &1555 EIP97_CFSIZE_MASK;1556 priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &1557 EIP97_RFSIZE_MASK;1558 priv->hwconfig.hwnumpes = 1; /* by definition */1559 priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &1560 EIP197_N_RINGS_MASK;1561 }1562 1563 /* Scan for ring AIC's */1564 for (i = 0; i < EIP197_MAX_RING_AIC; i++) {1565 version = readl(EIP197_HIA_AIC_R(priv) +1566 EIP197_HIA_AIC_R_VERSION(i));1567 if (EIP197_REG_LO16(version) != EIP201_VERSION_LE)1568 break;1569 }1570 priv->hwconfig.hwnumraic = i;1571 /* Low-end EIP196 may not have any ring AIC's ... */1572 if (!priv->hwconfig.hwnumraic) {1573 dev_err(priv->dev, "No ring interrupt controller present!\n");1574 return -ENODEV;1575 }1576 1577 /* Get supported algorithms from EIP96 transform engine */1578 priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +1579 EIP197_PE_EIP96_OPTIONS(0));1580 1581 /* Print single info line describing what we just detected */1582 dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n",1583 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes,1584 priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic,1585 priv->hwconfig.hiaver, priv->hwconfig.hwdataw,1586 priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize,1587 priv->hwconfig.ppver, priv->hwconfig.pever,1588 priv->hwconfig.algo_flags, priv->hwconfig.icever,1589 priv->hwconfig.ocever, priv->hwconfig.psever);1590 1591 safexcel_configure(priv);1592 1593 if (IS_ENABLED(CONFIG_PCI) && priv->data->version == EIP197_DEVBRD) {1594 /*1595 * Request MSI vectors for global + 1 per ring -1596 * or just 1 for older dev images1597 */1598 struct pci_dev *pci_pdev = pdev;1599 1600 ret = pci_alloc_irq_vectors(pci_pdev,1601 priv->config.rings + 1,1602 priv->config.rings + 1,1603 PCI_IRQ_MSI | PCI_IRQ_MSIX);1604 if (ret < 0) {1605 dev_err(dev, "Failed to allocate PCI MSI interrupts\n");1606 return ret;1607 }1608 }1609 1610 /* Register the ring IRQ handlers and configure the rings */1611 priv->ring = devm_kcalloc(dev, priv->config.rings,1612 sizeof(*priv->ring),1613 GFP_KERNEL);1614 if (!priv->ring)1615 return -ENOMEM;1616 1617 for (i = 0; i < priv->config.rings; i++) {1618 char wq_name[9] = {0};1619 int irq;1620 struct safexcel_ring_irq_data *ring_irq;1621 1622 ret = safexcel_init_ring_descriptors(priv,1623 &priv->ring[i].cdr,1624 &priv->ring[i].rdr);1625 if (ret) {1626 dev_err(dev, "Failed to initialize rings\n");1627 goto err_cleanup_rings;1628 }1629 1630 priv->ring[i].rdr_req = devm_kcalloc(dev,1631 EIP197_DEFAULT_RING_SIZE,1632 sizeof(*priv->ring[i].rdr_req),1633 GFP_KERNEL);1634 if (!priv->ring[i].rdr_req) {1635 ret = -ENOMEM;1636 goto err_cleanup_rings;1637 }1638 1639 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);1640 if (!ring_irq) {1641 ret = -ENOMEM;1642 goto err_cleanup_rings;1643 }1644 1645 ring_irq->priv = priv;1646 ring_irq->ring = i;1647 1648 irq = safexcel_request_ring_irq(pdev,1649 EIP197_IRQ_NUMBER(i, is_pci_dev),1650 is_pci_dev,1651 i,1652 safexcel_irq_ring,1653 safexcel_irq_ring_thread,1654 ring_irq);1655 if (irq < 0) {1656 dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);1657 ret = irq;1658 goto err_cleanup_rings;1659 }1660 1661 priv->ring[i].irq = irq;1662 priv->ring[i].work_data.priv = priv;1663 priv->ring[i].work_data.ring = i;1664 INIT_WORK(&priv->ring[i].work_data.work,1665 safexcel_dequeue_work);1666 1667 snprintf(wq_name, 9, "wq_ring%d", i);1668 priv->ring[i].workqueue =1669 create_singlethread_workqueue(wq_name);1670 if (!priv->ring[i].workqueue) {1671 ret = -ENOMEM;1672 goto err_cleanup_rings;1673 }1674 1675 priv->ring[i].requests = 0;1676 priv->ring[i].busy = false;1677 1678 crypto_init_queue(&priv->ring[i].queue,1679 EIP197_DEFAULT_RING_SIZE);1680 1681 spin_lock_init(&priv->ring[i].lock);1682 spin_lock_init(&priv->ring[i].queue_lock);1683 }1684 1685 atomic_set(&priv->ring_used, 0);1686 1687 ret = safexcel_hw_init(priv);1688 if (ret) {1689 dev_err(dev, "HW init failed (%d)\n", ret);1690 goto err_cleanup_rings;1691 }1692 1693 ret = safexcel_register_algorithms(priv);1694 if (ret) {1695 dev_err(dev, "Failed to register algorithms (%d)\n", ret);1696 goto err_cleanup_rings;1697 }1698 1699 return 0;1700 1701err_cleanup_rings:1702 for (i = 0; i < priv->config.rings; i++) {1703 if (priv->ring[i].irq)1704 irq_set_affinity_hint(priv->ring[i].irq, NULL);1705 if (priv->ring[i].workqueue)1706 destroy_workqueue(priv->ring[i].workqueue);1707 }1708 1709 return ret;1710}1711 1712static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)1713{1714 int i;1715 1716 for (i = 0; i < priv->config.rings; i++) {1717 /* clear any pending interrupt */1718 writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);1719 writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);1720 1721 /* Reset the CDR base address */1722 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);1723 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);1724 1725 /* Reset the RDR base address */1726 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);1727 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);1728 }1729}1730 1731/* for Device Tree platform driver */1732 1733static int safexcel_probe(struct platform_device *pdev)1734{1735 struct device *dev = &pdev->dev;1736 struct safexcel_crypto_priv *priv;1737 int ret;1738 1739 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);1740 if (!priv)1741 return -ENOMEM;1742 1743 priv->dev = dev;1744 priv->data = (struct safexcel_priv_data *)of_device_get_match_data(dev);1745 1746 platform_set_drvdata(pdev, priv);1747 1748 priv->base = devm_platform_ioremap_resource(pdev, 0);1749 if (IS_ERR(priv->base)) {1750 dev_err(dev, "failed to get resource\n");1751 return PTR_ERR(priv->base);1752 }1753 1754 priv->clk = devm_clk_get(&pdev->dev, NULL);1755 ret = PTR_ERR_OR_ZERO(priv->clk);1756 /* The clock isn't mandatory */1757 if (ret != -ENOENT) {1758 if (ret)1759 return ret;1760 1761 ret = clk_prepare_enable(priv->clk);1762 if (ret) {1763 dev_err(dev, "unable to enable clk (%d)\n", ret);1764 return ret;1765 }1766 }1767 1768 priv->reg_clk = devm_clk_get(&pdev->dev, "reg");1769 ret = PTR_ERR_OR_ZERO(priv->reg_clk);1770 /* The clock isn't mandatory */1771 if (ret != -ENOENT) {1772 if (ret)1773 goto err_core_clk;1774 1775 ret = clk_prepare_enable(priv->reg_clk);1776 if (ret) {1777 dev_err(dev, "unable to enable reg clk (%d)\n", ret);1778 goto err_core_clk;1779 }1780 }1781 1782 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));1783 if (ret)1784 goto err_reg_clk;1785 1786 /* Generic EIP97/EIP197 device probing */1787 ret = safexcel_probe_generic(pdev, priv, 0);1788 if (ret)1789 goto err_reg_clk;1790 1791 return 0;1792 1793err_reg_clk:1794 clk_disable_unprepare(priv->reg_clk);1795err_core_clk:1796 clk_disable_unprepare(priv->clk);1797 return ret;1798}1799 1800static void safexcel_remove(struct platform_device *pdev)1801{1802 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);1803 int i;1804 1805 safexcel_unregister_algorithms(priv);1806 safexcel_hw_reset_rings(priv);1807 1808 clk_disable_unprepare(priv->reg_clk);1809 clk_disable_unprepare(priv->clk);1810 1811 for (i = 0; i < priv->config.rings; i++) {1812 irq_set_affinity_hint(priv->ring[i].irq, NULL);1813 destroy_workqueue(priv->ring[i].workqueue);1814 }1815}1816 1817static const struct safexcel_priv_data eip97ies_mrvl_data = {1818 .version = EIP97IES_MRVL,1819};1820 1821static const struct safexcel_priv_data eip197b_mrvl_data = {1822 .version = EIP197B_MRVL,1823};1824 1825static const struct safexcel_priv_data eip197d_mrvl_data = {1826 .version = EIP197D_MRVL,1827};1828 1829static const struct safexcel_priv_data eip197_devbrd_data = {1830 .version = EIP197_DEVBRD,1831};1832 1833static const struct safexcel_priv_data eip197c_mxl_data = {1834 .version = EIP197C_MXL,1835 .fw_little_endian = true,1836};1837 1838static const struct of_device_id safexcel_of_match_table[] = {1839 {1840 .compatible = "inside-secure,safexcel-eip97ies",1841 .data = &eip97ies_mrvl_data,1842 },1843 {1844 .compatible = "inside-secure,safexcel-eip197b",1845 .data = &eip197b_mrvl_data,1846 },1847 {1848 .compatible = "inside-secure,safexcel-eip197d",1849 .data = &eip197d_mrvl_data,1850 },1851 {1852 .compatible = "inside-secure,safexcel-eip197c-mxl",1853 .data = &eip197c_mxl_data,1854 },1855 /* For backward compatibility and intended for generic use */1856 {1857 .compatible = "inside-secure,safexcel-eip97",1858 .data = &eip97ies_mrvl_data,1859 },1860 {1861 .compatible = "inside-secure,safexcel-eip197",1862 .data = &eip197b_mrvl_data,1863 },1864 {},1865};1866 1867MODULE_DEVICE_TABLE(of, safexcel_of_match_table);1868 1869static struct platform_driver crypto_safexcel = {1870 .probe = safexcel_probe,1871 .remove_new = safexcel_remove,1872 .driver = {1873 .name = "crypto-safexcel",1874 .of_match_table = safexcel_of_match_table,1875 },1876};1877 1878/* PCIE devices - i.e. Inside Secure development boards */1879 1880static int safexcel_pci_probe(struct pci_dev *pdev,1881 const struct pci_device_id *ent)1882{1883 struct device *dev = &pdev->dev;1884 struct safexcel_crypto_priv *priv;1885 void __iomem *pciebase;1886 int rc;1887 u32 val;1888 1889 dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",1890 ent->vendor, ent->device, ent->subvendor,1891 ent->subdevice, ent->driver_data);1892 1893 priv = kzalloc(sizeof(*priv), GFP_KERNEL);1894 if (!priv)1895 return -ENOMEM;1896 1897 priv->dev = dev;1898 priv->data = (struct safexcel_priv_data *)ent->driver_data;1899 1900 pci_set_drvdata(pdev, priv);1901 1902 /* enable the device */1903 rc = pcim_enable_device(pdev);1904 if (rc) {1905 dev_err(dev, "Failed to enable PCI device\n");1906 return rc;1907 }1908 1909 /* take ownership of PCI BAR0 */1910 rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");1911 if (rc) {1912 dev_err(dev, "Failed to map IO region for BAR0\n");1913 return rc;1914 }1915 priv->base = pcim_iomap_table(pdev)[0];1916 1917 if (priv->data->version == EIP197_DEVBRD) {1918 dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");1919 1920 rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");1921 if (rc) {1922 dev_err(dev, "Failed to map IO region for BAR4\n");1923 return rc;1924 }1925 1926 pciebase = pcim_iomap_table(pdev)[2];1927 val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);1928 if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {1929 dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",1930 (val & 0xff));1931 1932 /* Setup MSI identity map mapping */1933 writel(EIP197_XLX_USER_VECT_LUT0_IDENT,1934 pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);1935 writel(EIP197_XLX_USER_VECT_LUT1_IDENT,1936 pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);1937 writel(EIP197_XLX_USER_VECT_LUT2_IDENT,1938 pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);1939 writel(EIP197_XLX_USER_VECT_LUT3_IDENT,1940 pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);1941 1942 /* Enable all device interrupts */1943 writel(GENMASK(31, 0),1944 pciebase + EIP197_XLX_USER_INT_ENB_MSK);1945 } else {1946 dev_err(dev, "Unrecognised IRQ block identifier %x\n",1947 val);1948 return -ENODEV;1949 }1950 1951 /* HW reset FPGA dev board */1952 /* assert reset */1953 writel(1, priv->base + EIP197_XLX_GPIO_BASE);1954 wmb(); /* maintain strict ordering for accesses here */1955 /* deassert reset */1956 writel(0, priv->base + EIP197_XLX_GPIO_BASE);1957 wmb(); /* maintain strict ordering for accesses here */1958 }1959 1960 /* enable bus mastering */1961 pci_set_master(pdev);1962 1963 /* Generic EIP97/EIP197 device probing */1964 rc = safexcel_probe_generic(pdev, priv, 1);1965 return rc;1966}1967 1968static void safexcel_pci_remove(struct pci_dev *pdev)1969{1970 struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);1971 int i;1972 1973 safexcel_unregister_algorithms(priv);1974 1975 for (i = 0; i < priv->config.rings; i++)1976 destroy_workqueue(priv->ring[i].workqueue);1977 1978 safexcel_hw_reset_rings(priv);1979}1980 1981static const struct pci_device_id safexcel_pci_ids[] = {1982 {1983 PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,1984 0x16ae, 0xc522),1985 .driver_data = (kernel_ulong_t)&eip197_devbrd_data,1986 },1987 {},1988};1989 1990MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);1991 1992static struct pci_driver safexcel_pci_driver = {1993 .name = "crypto-safexcel",1994 .id_table = safexcel_pci_ids,1995 .probe = safexcel_pci_probe,1996 .remove = safexcel_pci_remove,1997};1998 1999static int __init safexcel_init(void)2000{2001 int ret;2002 2003 /* Register PCI driver */2004 ret = pci_register_driver(&safexcel_pci_driver);2005 2006 /* Register platform driver */2007 if (IS_ENABLED(CONFIG_OF) && !ret) {2008 ret = platform_driver_register(&crypto_safexcel);2009 if (ret)2010 pci_unregister_driver(&safexcel_pci_driver);2011 }2012 2013 return ret;2014}2015 2016static void __exit safexcel_exit(void)2017{2018 /* Unregister platform driver */2019 if (IS_ENABLED(CONFIG_OF))2020 platform_driver_unregister(&crypto_safexcel);2021 2022 /* Unregister PCI driver if successfully registered before */2023 pci_unregister_driver(&safexcel_pci_driver);2024}2025 2026module_init(safexcel_init);2027module_exit(safexcel_exit);2028 2029MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");2030MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");2031MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");2032MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");2033MODULE_LICENSE("GPL v2");2034MODULE_IMPORT_NS(CRYPTO_INTERNAL);2035 2036MODULE_FIRMWARE("ifpp.bin");2037MODULE_FIRMWARE("ipue.bin");2038MODULE_FIRMWARE("inside-secure/eip197b/ifpp.bin");2039MODULE_FIRMWARE("inside-secure/eip197b/ipue.bin");2040MODULE_FIRMWARE("inside-secure/eip197d/ifpp.bin");2041MODULE_FIRMWARE("inside-secure/eip197d/ipue.bin");2042MODULE_FIRMWARE("inside-secure/eip197_minifw/ifpp.bin");2043MODULE_FIRMWARE("inside-secure/eip197_minifw/ipue.bin");2044