2563 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>4 * All rights reserved.5 */6 7#include <linux/kernel.h>8#include <linux/module.h>9#include <linux/moduleparam.h>10#include <linux/mod_devicetable.h>11#include <linux/interrupt.h>12#include <linux/pci.h>13#include <linux/slab.h>14#include <linux/delay.h>15#include <linux/mm.h>16#include <linux/dma-mapping.h>17#include <linux/scatterlist.h>18#include <linux/highmem.h>19#include <linux/crypto.h>20#include <linux/hw_random.h>21#include <linux/ktime.h>22 23#include <crypto/algapi.h>24#include <crypto/internal/des.h>25#include <crypto/internal/skcipher.h>26 27static char hifn_pll_ref[sizeof("extNNN")] = "ext";28module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);29MODULE_PARM_DESC(hifn_pll_ref,30 "PLL reference clock (pci[freq] or ext[freq], default ext)");31 32static atomic_t hifn_dev_number;33 34#define ACRYPTO_OP_DECRYPT 035#define ACRYPTO_OP_ENCRYPT 136#define ACRYPTO_OP_HMAC 237#define ACRYPTO_OP_RNG 338 39#define ACRYPTO_MODE_ECB 040#define ACRYPTO_MODE_CBC 141#define ACRYPTO_MODE_CFB 242#define ACRYPTO_MODE_OFB 343 44#define ACRYPTO_TYPE_AES_128 045#define ACRYPTO_TYPE_AES_192 146#define ACRYPTO_TYPE_AES_256 247#define ACRYPTO_TYPE_3DES 348#define ACRYPTO_TYPE_DES 449 50#define PCI_VENDOR_ID_HIFN 0x13A351#define PCI_DEVICE_ID_HIFN_7955 0x002052#define PCI_DEVICE_ID_HIFN_7956 0x001d53 54/* I/O region sizes */55 56#define HIFN_BAR0_SIZE 0x100057#define HIFN_BAR1_SIZE 0x200058#define HIFN_BAR2_SIZE 0x800059 60/* DMA registres */61 62#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */63#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */64#define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */65#define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */66#define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */67#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */68#define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */69#define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */70#define HIFN_CHIP_ID 0x98 /* Chip ID */71 72/*73 * Processing Unit Registers (offset from BASEREG0)74 */75#define HIFN_0_PUDATA 0x00 /* Processing Unit Data */76#define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */77#define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */78#define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */79#define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */80#define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */81#define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */82#define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */83#define HIFN_0_SPACESIZE 0x20 /* Register space size */84 85/* Processing Unit Control Register (HIFN_0_PUCTRL) */86#define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */87#define HIFN_PUCTRL_STOP 0x0008 /* stop pu */88#define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */89#define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */90#define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */91 92/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */93#define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */94#define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */95#define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */96#define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */97#define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */98#define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */99#define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */100#define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */101#define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */102#define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */103 104/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */105#define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */106#define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */107#define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */108#define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */109#define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */110#define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */111#define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */112#define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */113#define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */114#define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */115#define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */116#define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */117#define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */118#define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */119#define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */120#define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */121#define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */122#define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */123#define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */124#define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */125#define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */126#define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */127#define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */128 129/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */130#define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */131#define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */132#define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */133#define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */134#define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */135#define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */136#define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */137#define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */138#define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */139#define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */140 141/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */142#define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */143#define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */144#define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */145#define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */146#define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */147#define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */148#define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */149#define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */150#define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */151#define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */152#define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */153#define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */154#define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */155#define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */156#define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */157#define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */158#define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */159 160/* FIFO Status Register (HIFN_0_FIFOSTAT) */161#define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */162#define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */163 164/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */165#define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */166 167/*168 * DMA Interface Registers (offset from BASEREG1)169 */170#define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */171#define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */172#define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */173#define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */174#define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */175#define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */176#define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */177#define HIFN_1_PLL 0x4c /* 795x: PLL config */178#define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */179#define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */180#define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */181#define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */182#define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */183#define HIFN_1_REVID 0x98 /* Revision ID */184#define HIFN_1_UNLOCK_SECRET1 0xf4185#define HIFN_1_UNLOCK_SECRET2 0xfc186#define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */187#define HIFN_1_PUB_BASE 0x300 /* Public Base Address */188#define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */189#define HIFN_1_PUB_OP 0x308 /* Public Operand */190#define HIFN_1_PUB_STATUS 0x30c /* Public Status */191#define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */192#define HIFN_1_RNG_CONFIG 0x314 /* RNG config */193#define HIFN_1_RNG_DATA 0x318 /* RNG data */194#define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */195#define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */196 197/* DMA Status and Control Register (HIFN_1_DMA_CSR) */198#define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */199#define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */200#define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */201#define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */202#define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */203#define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */204#define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */205#define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */206#define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */207#define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */208#define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */209#define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */210#define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */211#define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */212#define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */213#define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */214#define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */215#define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */216#define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */217#define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */218#define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */219#define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */220#define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */221#define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */222#define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */223#define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */224#define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */225#define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */226#define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */227#define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */228#define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */229#define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */230#define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */231#define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */232#define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */233#define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */234#define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */235#define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */236 237/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */238#define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */239#define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */240#define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */241#define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */242#define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */243#define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */244#define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */245#define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */246#define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */247#define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */248#define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */249#define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */250#define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */251#define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */252#define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */253#define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */254#define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */255#define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */256#define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */257#define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */258#define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */259#define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */260 261/* DMA Configuration Register (HIFN_1_DMA_CNFG) */262#define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */263#define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */264#define HIFN_DMACNFG_UNLOCK 0x00000800265#define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */266#define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */267#define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */268#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */269#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */270 271/* PLL configuration register */272#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */273#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */274#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */275#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */276#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */277#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */278#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */279#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */280#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */281#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */282#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */283#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */284#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */285#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */286#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */287#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */288#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */289 290#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */291 292/* Public key reset register (HIFN_1_PUB_RESET) */293#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */294 295/* Public base address register (HIFN_1_PUB_BASE) */296#define HIFN_PUBBASE_ADDR 0x00003fff /* base address */297 298/* Public operand length register (HIFN_1_PUB_OPLEN) */299#define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */300#define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */301#define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */302#define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */303#define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */304#define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */305 306/* Public operation register (HIFN_1_PUB_OP) */307#define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */308#define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */309#define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */310#define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */311#define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */312#define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */313#define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */314#define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */315#define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */316#define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */317#define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */318#define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */319#define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */320#define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */321#define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */322#define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */323#define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */324#define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */325#define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */326#define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */327 328/* Public status register (HIFN_1_PUB_STATUS) */329#define HIFN_PUBSTS_DONE 0x00000001 /* operation done */330#define HIFN_PUBSTS_CARRY 0x00000002 /* carry */331 332/* Public interrupt enable register (HIFN_1_PUB_IEN) */333#define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */334 335/* Random number generator config register (HIFN_1_RNG_CONFIG) */336#define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */337 338#define HIFN_NAMESIZE 32339#define HIFN_MAX_RESULT_ORDER 5340 341#define HIFN_D_CMD_RSIZE (24 * 1)342#define HIFN_D_SRC_RSIZE (80 * 1)343#define HIFN_D_DST_RSIZE (80 * 1)344#define HIFN_D_RES_RSIZE (24 * 1)345 346#define HIFN_D_DST_DALIGN 4347 348#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1)349 350#define AES_MIN_KEY_SIZE 16351#define AES_MAX_KEY_SIZE 32352 353#define HIFN_DES_KEY_LENGTH 8354#define HIFN_3DES_KEY_LENGTH 24355#define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE356#define HIFN_IV_LENGTH 8357#define HIFN_AES_IV_LENGTH 16358#define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH359 360#define HIFN_MAC_KEY_LENGTH 64361#define HIFN_MD5_LENGTH 16362#define HIFN_SHA1_LENGTH 20363#define HIFN_MAC_TRUNC_LENGTH 12364 365#define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260)366#define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4)367#define HIFN_USED_RESULT 12368 369struct hifn_desc {370 volatile __le32 l;371 volatile __le32 p;372};373 374struct hifn_dma {375 struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1];376 struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1];377 struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1];378 struct hifn_desc resr[HIFN_D_RES_RSIZE + 1];379 380 u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];381 u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];382 383 /*384 * Our current positions for insertion and removal from the descriptor385 * rings.386 */387 volatile int cmdi, srci, dsti, resi;388 volatile int cmdu, srcu, dstu, resu;389 int cmdk, srck, dstk, resk;390};391 392#define HIFN_FLAG_CMD_BUSY (1 << 0)393#define HIFN_FLAG_SRC_BUSY (1 << 1)394#define HIFN_FLAG_DST_BUSY (1 << 2)395#define HIFN_FLAG_RES_BUSY (1 << 3)396#define HIFN_FLAG_OLD_KEY (1 << 4)397 398#define HIFN_DEFAULT_ACTIVE_NUM 5399 400struct hifn_device {401 char name[HIFN_NAMESIZE];402 403 int irq;404 405 struct pci_dev *pdev;406 void __iomem *bar[3];407 408 void *desc_virt;409 dma_addr_t desc_dma;410 411 u32 dmareg;412 413 void *sa[HIFN_D_RES_RSIZE];414 415 spinlock_t lock;416 417 u32 flags;418 int active, started;419 struct delayed_work work;420 unsigned long reset;421 unsigned long success;422 unsigned long prev_success;423 424 u8 snum;425 426 struct tasklet_struct tasklet;427 428 struct crypto_queue queue;429 struct list_head alg_list;430 431 unsigned int pk_clk_freq;432 433#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG434 unsigned int rng_wait_time;435 ktime_t rngtime;436 struct hwrng rng;437#endif438};439 440#define HIFN_D_LENGTH 0x0000ffff441#define HIFN_D_NOINVALID 0x01000000442#define HIFN_D_MASKDONEIRQ 0x02000000443#define HIFN_D_DESTOVER 0x04000000444#define HIFN_D_OVER 0x08000000445#define HIFN_D_LAST 0x20000000446#define HIFN_D_JUMP 0x40000000447#define HIFN_D_VALID 0x80000000448 449struct hifn_base_command {450 volatile __le16 masks;451 volatile __le16 session_num;452 volatile __le16 total_source_count;453 volatile __le16 total_dest_count;454};455 456#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */457#define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */458#define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */459#define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */460#define HIFN_BASE_CMD_DECODE 0x2000461#define HIFN_BASE_CMD_SRCLEN_M 0xc000462#define HIFN_BASE_CMD_SRCLEN_S 14463#define HIFN_BASE_CMD_DSTLEN_M 0x3000464#define HIFN_BASE_CMD_DSTLEN_S 12465#define HIFN_BASE_CMD_LENMASK_HI 0x30000466#define HIFN_BASE_CMD_LENMASK_LO 0x0ffff467 468/*469 * Structure to help build up the command data structure.470 */471struct hifn_crypt_command {472 volatile __le16 masks;473 volatile __le16 header_skip;474 volatile __le16 source_count;475 volatile __le16 reserved;476};477 478#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */479#define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */480#define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */481#define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */482#define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */483#define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */484#define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */485#define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */486#define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */487#define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */488#define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */489#define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */490#define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */491#define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */492#define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */493#define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */494#define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */495#define HIFN_CRYPT_CMD_SRCLEN_M 0xc000496#define HIFN_CRYPT_CMD_SRCLEN_S 14497 498#define HIFN_MAC_CMD_ALG_MASK 0x0001499#define HIFN_MAC_CMD_ALG_SHA1 0x0000500#define HIFN_MAC_CMD_ALG_MD5 0x0001501#define HIFN_MAC_CMD_MODE_MASK 0x000c502#define HIFN_MAC_CMD_MODE_HMAC 0x0000503#define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004504#define HIFN_MAC_CMD_MODE_HASH 0x0008505#define HIFN_MAC_CMD_MODE_FULL 0x0004506#define HIFN_MAC_CMD_TRUNC 0x0010507#define HIFN_MAC_CMD_RESULT 0x0020508#define HIFN_MAC_CMD_APPEND 0x0040509#define HIFN_MAC_CMD_SRCLEN_M 0xc000510#define HIFN_MAC_CMD_SRCLEN_S 14511 512/*513 * MAC POS IPsec initiates authentication after encryption on encodes514 * and before decryption on decodes.515 */516#define HIFN_MAC_CMD_POS_IPSEC 0x0200517#define HIFN_MAC_CMD_NEW_KEY 0x0800518 519#define HIFN_COMP_CMD_SRCLEN_M 0xc000520#define HIFN_COMP_CMD_SRCLEN_S 14521#define HIFN_COMP_CMD_ONE 0x0100 /* must be one */522#define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */523#define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */524#define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */525#define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */526#define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */527#define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */528#define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */529 530struct hifn_base_result {531 volatile __le16 flags;532 volatile __le16 session;533 volatile __le16 src_cnt; /* 15:0 of source count */534 volatile __le16 dst_cnt; /* 15:0 of dest count */535};536 537#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */538#define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */539#define HIFN_BASE_RES_SRCLEN_S 14540#define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */541#define HIFN_BASE_RES_DSTLEN_S 12542 543struct hifn_comp_result {544 volatile __le16 flags;545 volatile __le16 crc;546};547 548#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */549#define HIFN_COMP_RES_LCB_S 8550#define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */551#define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */552#define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */553 554struct hifn_mac_result {555 volatile __le16 flags;556 volatile __le16 reserved;557 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */558};559 560#define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */561#define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */562 563struct hifn_crypt_result {564 volatile __le16 flags;565 volatile __le16 reserved;566};567 568#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */569 570#ifndef HIFN_POLL_FREQUENCY571#define HIFN_POLL_FREQUENCY 0x1572#endif573 574#ifndef HIFN_POLL_SCALAR575#define HIFN_POLL_SCALAR 0x0576#endif577 578#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */579#define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */580 581struct hifn_crypto_alg {582 struct list_head entry;583 struct skcipher_alg alg;584 struct hifn_device *dev;585};586 587#define ASYNC_SCATTERLIST_CACHE 16588 589#define ASYNC_FLAGS_MISALIGNED (1 << 0)590 591struct hifn_cipher_walk {592 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE];593 u32 flags;594 int num;595};596 597struct hifn_context {598 u8 key[HIFN_MAX_CRYPT_KEY_LENGTH];599 struct hifn_device *dev;600 unsigned int keysize;601};602 603struct hifn_request_context {604 u8 *iv;605 unsigned int ivsize;606 u8 op, type, mode, unused;607 struct hifn_cipher_walk walk;608};609 610#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)611 612static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)613{614 return readl(dev->bar[0] + reg);615}616 617static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)618{619 return readl(dev->bar[1] + reg);620}621 622static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)623{624 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);625}626 627static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)628{629 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);630}631 632static void hifn_wait_puc(struct hifn_device *dev)633{634 int i;635 u32 ret;636 637 for (i = 10000; i > 0; --i) {638 ret = hifn_read_0(dev, HIFN_0_PUCTRL);639 if (!(ret & HIFN_PUCTRL_RESET))640 break;641 642 udelay(1);643 }644 645 if (!i)646 dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n");647}648 649static void hifn_reset_puc(struct hifn_device *dev)650{651 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);652 hifn_wait_puc(dev);653}654 655static void hifn_stop_device(struct hifn_device *dev)656{657 hifn_write_1(dev, HIFN_1_DMA_CSR,658 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |659 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);660 hifn_write_0(dev, HIFN_0_PUIER, 0);661 hifn_write_1(dev, HIFN_1_DMA_IER, 0);662}663 664static void hifn_reset_dma(struct hifn_device *dev, int full)665{666 hifn_stop_device(dev);667 668 /*669 * Setting poll frequency and others to 0.670 */671 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |672 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);673 mdelay(1);674 675 /*676 * Reset DMA.677 */678 if (full) {679 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);680 mdelay(1);681 } else {682 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |683 HIFN_DMACNFG_MSTRESET);684 hifn_reset_puc(dev);685 }686 687 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |688 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);689 690 hifn_reset_puc(dev);691}692 693static u32 hifn_next_signature(u32 a, u_int cnt)694{695 int i;696 u32 v;697 698 for (i = 0; i < cnt; i++) {699 /* get the parity */700 v = a & 0x80080125;701 v ^= v >> 16;702 v ^= v >> 8;703 v ^= v >> 4;704 v ^= v >> 2;705 v ^= v >> 1;706 707 a = (v & 1) ^ (a << 1);708 }709 710 return a;711}712 713static struct pci2id {714 u_short pci_vendor;715 u_short pci_prod;716 char card_id[13];717} pci2id[] = {718 {719 PCI_VENDOR_ID_HIFN,720 PCI_DEVICE_ID_HIFN_7955,721 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,722 0x00, 0x00, 0x00, 0x00, 0x00 }723 },724 {725 PCI_VENDOR_ID_HIFN,726 PCI_DEVICE_ID_HIFN_7956,727 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,728 0x00, 0x00, 0x00, 0x00, 0x00 }729 }730};731 732#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG733static int hifn_rng_data_present(struct hwrng *rng, int wait)734{735 struct hifn_device *dev = (struct hifn_device *)rng->priv;736 s64 nsec;737 738 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));739 nsec -= dev->rng_wait_time;740 if (nsec <= 0)741 return 1;742 if (!wait)743 return 0;744 ndelay(nsec);745 return 1;746}747 748static int hifn_rng_data_read(struct hwrng *rng, u32 *data)749{750 struct hifn_device *dev = (struct hifn_device *)rng->priv;751 752 *data = hifn_read_1(dev, HIFN_1_RNG_DATA);753 dev->rngtime = ktime_get();754 return 4;755}756 757static int hifn_register_rng(struct hifn_device *dev)758{759 /*760 * We must wait at least 256 Pk_clk cycles between two reads of the rng.761 */762 dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC,763 dev->pk_clk_freq) * 256;764 765 dev->rng.name = dev->name;766 dev->rng.data_present = hifn_rng_data_present;767 dev->rng.data_read = hifn_rng_data_read;768 dev->rng.priv = (unsigned long)dev;769 770 return hwrng_register(&dev->rng);771}772 773static void hifn_unregister_rng(struct hifn_device *dev)774{775 hwrng_unregister(&dev->rng);776}777#else778#define hifn_register_rng(dev) 0779#define hifn_unregister_rng(dev)780#endif781 782static int hifn_init_pubrng(struct hifn_device *dev)783{784 int i;785 786 hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |787 HIFN_PUBRST_RESET);788 789 for (i = 100; i > 0; --i) {790 mdelay(1);791 792 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)793 break;794 }795 796 if (!i) {797 dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n");798 } else {799 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);800 dev->dmareg |= HIFN_DMAIER_PUBDONE;801 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);802 803 dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n");804 }805 806 /* Enable RNG engine. */807 808 hifn_write_1(dev, HIFN_1_RNG_CONFIG,809 hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);810 dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n");811 812#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG813 /* First value must be discarded */814 hifn_read_1(dev, HIFN_1_RNG_DATA);815 dev->rngtime = ktime_get();816#endif817 return 0;818}819 820static int hifn_enable_crypto(struct hifn_device *dev)821{822 u32 dmacfg, addr;823 char *offtbl = NULL;824 int i;825 826 for (i = 0; i < ARRAY_SIZE(pci2id); i++) {827 if (pci2id[i].pci_vendor == dev->pdev->vendor &&828 pci2id[i].pci_prod == dev->pdev->device) {829 offtbl = pci2id[i].card_id;830 break;831 }832 }833 834 if (!offtbl) {835 dev_err(&dev->pdev->dev, "Unknown card!\n");836 return -ENODEV;837 }838 839 dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);840 841 hifn_write_1(dev, HIFN_1_DMA_CNFG,842 HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |843 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);844 mdelay(1);845 addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);846 mdelay(1);847 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);848 mdelay(1);849 850 for (i = 0; i < 12; ++i) {851 addr = hifn_next_signature(addr, offtbl[i] + 0x101);852 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);853 854 mdelay(1);855 }856 hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);857 858 dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev));859 860 return 0;861}862 863static void hifn_init_dma(struct hifn_device *dev)864{865 struct hifn_dma *dma = dev->desc_virt;866 u32 dptr = dev->desc_dma;867 int i;868 869 for (i = 0; i < HIFN_D_CMD_RSIZE; ++i)870 dma->cmdr[i].p = __cpu_to_le32(dptr +871 offsetof(struct hifn_dma, command_bufs[i][0]));872 for (i = 0; i < HIFN_D_RES_RSIZE; ++i)873 dma->resr[i].p = __cpu_to_le32(dptr +874 offsetof(struct hifn_dma, result_bufs[i][0]));875 876 /* Setup LAST descriptors. */877 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +878 offsetof(struct hifn_dma, cmdr[0]));879 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +880 offsetof(struct hifn_dma, srcr[0]));881 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +882 offsetof(struct hifn_dma, dstr[0]));883 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +884 offsetof(struct hifn_dma, resr[0]));885 886 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;887 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;888 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;889}890 891/*892 * Initialize the PLL. We need to know the frequency of the reference clock893 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that894 * allows us to operate without the risk of overclocking the chip. If it895 * actually uses 33MHz, the chip will operate at half the speed, this can be896 * overridden by specifying the frequency as module parameter (pci33).897 *898 * Unfortunately the PCI clock is not very suitable since the HIFN needs a899 * stable clock and the PCI clock frequency may vary, so the default is the900 * external clock. There is no way to find out its frequency, we default to901 * 66MHz since according to Mike Ham of HiFn, almost every board in existence902 * has an external crystal populated at 66MHz.903 */904static void hifn_init_pll(struct hifn_device *dev)905{906 unsigned int freq, m;907 u32 pllcfg;908 909 pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;910 911 if (strncmp(hifn_pll_ref, "ext", 3) == 0)912 pllcfg |= HIFN_PLL_REF_CLK_PLL;913 else914 pllcfg |= HIFN_PLL_REF_CLK_HBI;915 916 if (hifn_pll_ref[3] != '\0')917 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);918 else {919 freq = 66;920 dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n",921 freq, hifn_pll_ref);922 }923 924 m = HIFN_PLL_FCK_MAX / freq;925 926 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;927 if (m <= 8)928 pllcfg |= HIFN_PLL_IS_1_8;929 else930 pllcfg |= HIFN_PLL_IS_9_12;931 932 /* Select clock source and enable clock bypass */933 hifn_write_1(dev, HIFN_1_PLL, pllcfg |934 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);935 936 /* Let the chip lock to the input clock */937 mdelay(10);938 939 /* Disable clock bypass */940 hifn_write_1(dev, HIFN_1_PLL, pllcfg |941 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);942 943 /* Switch the engines to the PLL */944 hifn_write_1(dev, HIFN_1_PLL, pllcfg |945 HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);946 947 /*948 * The Fpk_clk runs at half the total speed. Its frequency is needed to949 * calculate the minimum time between two reads of the rng. Since 33MHz950 * is actually 33.333... we overestimate the frequency here, resulting951 * in slightly larger intervals.952 */953 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;954}955 956static void hifn_init_registers(struct hifn_device *dev)957{958 u32 dptr = dev->desc_dma;959 960 /* Initialization magic... */961 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);962 hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);963 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);964 965 /* write all 4 ring address registers */966 hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +967 offsetof(struct hifn_dma, cmdr[0]));968 hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +969 offsetof(struct hifn_dma, srcr[0]));970 hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +971 offsetof(struct hifn_dma, dstr[0]));972 hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +973 offsetof(struct hifn_dma, resr[0]));974 975 mdelay(2);976#if 0977 hifn_write_1(dev, HIFN_1_DMA_CSR,978 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |979 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |980 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |981 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |982 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |983 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |984 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |985 HIFN_DMACSR_S_WAIT |986 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |987 HIFN_DMACSR_C_WAIT |988 HIFN_DMACSR_ENGINE |989 HIFN_DMACSR_PUBDONE);990#else991 hifn_write_1(dev, HIFN_1_DMA_CSR,992 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |993 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |994 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |995 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |996 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |997 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |998 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |999 HIFN_DMACSR_S_WAIT |1000 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |1001 HIFN_DMACSR_C_WAIT |1002 HIFN_DMACSR_ENGINE |1003 HIFN_DMACSR_PUBDONE);1004#endif1005 hifn_read_1(dev, HIFN_1_DMA_CSR);1006 1007 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |1008 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |1009 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |1010 HIFN_DMAIER_ENGINE;1011 dev->dmareg &= ~HIFN_DMAIER_C_WAIT;1012 1013 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);1014 hifn_read_1(dev, HIFN_1_DMA_IER);1015#if 01016 hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |1017 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |1018 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |1019 HIFN_PUCNFG_DRAM);1020#else1021 hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);1022#endif1023 hifn_init_pll(dev);1024 1025 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);1026 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |1027 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |1028 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |1029 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));1030}1031 1032static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,1033 unsigned dlen, unsigned slen, u16 mask, u8 snum)1034{1035 struct hifn_base_command *base_cmd;1036 u8 *buf_pos = buf;1037 1038 base_cmd = (struct hifn_base_command *)buf_pos;1039 base_cmd->masks = __cpu_to_le16(mask);1040 base_cmd->total_source_count =1041 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);1042 base_cmd->total_dest_count =1043 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);1044 1045 dlen >>= 16;1046 slen >>= 16;1047 base_cmd->session_num = __cpu_to_le16(snum |1048 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |1049 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));1050 1051 return sizeof(struct hifn_base_command);1052}1053 1054static int hifn_setup_crypto_command(struct hifn_device *dev,1055 u8 *buf, unsigned dlen, unsigned slen,1056 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)1057{1058 struct hifn_dma *dma = dev->desc_virt;1059 struct hifn_crypt_command *cry_cmd;1060 u8 *buf_pos = buf;1061 u16 cmd_len;1062 1063 cry_cmd = (struct hifn_crypt_command *)buf_pos;1064 1065 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);1066 dlen >>= 16;1067 cry_cmd->masks = __cpu_to_le16(mode |1068 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &1069 HIFN_CRYPT_CMD_SRCLEN_M));1070 cry_cmd->header_skip = 0;1071 cry_cmd->reserved = 0;1072 1073 buf_pos += sizeof(struct hifn_crypt_command);1074 1075 dma->cmdu++;1076 if (dma->cmdu > 1) {1077 dev->dmareg |= HIFN_DMAIER_C_WAIT;1078 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);1079 }1080 1081 if (keylen) {1082 memcpy(buf_pos, key, keylen);1083 buf_pos += keylen;1084 }1085 if (ivsize) {1086 memcpy(buf_pos, iv, ivsize);1087 buf_pos += ivsize;1088 }1089 1090 cmd_len = buf_pos - buf;1091 1092 return cmd_len;1093}1094 1095static int hifn_setup_cmd_desc(struct hifn_device *dev,1096 struct hifn_context *ctx, struct hifn_request_context *rctx,1097 void *priv, unsigned int nbytes)1098{1099 struct hifn_dma *dma = dev->desc_virt;1100 int cmd_len, sa_idx;1101 u8 *buf, *buf_pos;1102 u16 mask;1103 1104 sa_idx = dma->cmdi;1105 buf_pos = buf = dma->command_bufs[dma->cmdi];1106 1107 mask = 0;1108 switch (rctx->op) {1109 case ACRYPTO_OP_DECRYPT:1110 mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;1111 break;1112 case ACRYPTO_OP_ENCRYPT:1113 mask = HIFN_BASE_CMD_CRYPT;1114 break;1115 case ACRYPTO_OP_HMAC:1116 mask = HIFN_BASE_CMD_MAC;1117 break;1118 default:1119 goto err_out;1120 }1121 1122 buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,1123 nbytes, mask, dev->snum);1124 1125 if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {1126 u16 md = 0;1127 1128 if (ctx->keysize)1129 md |= HIFN_CRYPT_CMD_NEW_KEY;1130 if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)1131 md |= HIFN_CRYPT_CMD_NEW_IV;1132 1133 switch (rctx->mode) {1134 case ACRYPTO_MODE_ECB:1135 md |= HIFN_CRYPT_CMD_MODE_ECB;1136 break;1137 case ACRYPTO_MODE_CBC:1138 md |= HIFN_CRYPT_CMD_MODE_CBC;1139 break;1140 case ACRYPTO_MODE_CFB:1141 md |= HIFN_CRYPT_CMD_MODE_CFB;1142 break;1143 case ACRYPTO_MODE_OFB:1144 md |= HIFN_CRYPT_CMD_MODE_OFB;1145 break;1146 default:1147 goto err_out;1148 }1149 1150 switch (rctx->type) {1151 case ACRYPTO_TYPE_AES_128:1152 if (ctx->keysize != 16)1153 goto err_out;1154 md |= HIFN_CRYPT_CMD_KSZ_128 |1155 HIFN_CRYPT_CMD_ALG_AES;1156 break;1157 case ACRYPTO_TYPE_AES_192:1158 if (ctx->keysize != 24)1159 goto err_out;1160 md |= HIFN_CRYPT_CMD_KSZ_192 |1161 HIFN_CRYPT_CMD_ALG_AES;1162 break;1163 case ACRYPTO_TYPE_AES_256:1164 if (ctx->keysize != 32)1165 goto err_out;1166 md |= HIFN_CRYPT_CMD_KSZ_256 |1167 HIFN_CRYPT_CMD_ALG_AES;1168 break;1169 case ACRYPTO_TYPE_3DES:1170 if (ctx->keysize != 24)1171 goto err_out;1172 md |= HIFN_CRYPT_CMD_ALG_3DES;1173 break;1174 case ACRYPTO_TYPE_DES:1175 if (ctx->keysize != 8)1176 goto err_out;1177 md |= HIFN_CRYPT_CMD_ALG_DES;1178 break;1179 default:1180 goto err_out;1181 }1182 1183 buf_pos += hifn_setup_crypto_command(dev, buf_pos,1184 nbytes, nbytes, ctx->key, ctx->keysize,1185 rctx->iv, rctx->ivsize, md);1186 }1187 1188 dev->sa[sa_idx] = priv;1189 dev->started++;1190 1191 cmd_len = buf_pos - buf;1192 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |1193 HIFN_D_LAST | HIFN_D_MASKDONEIRQ);1194 1195 if (++dma->cmdi == HIFN_D_CMD_RSIZE) {1196 dma->cmdr[dma->cmdi].l = __cpu_to_le32(1197 HIFN_D_VALID | HIFN_D_LAST |1198 HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);1199 dma->cmdi = 0;1200 } else {1201 dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID);1202 }1203 1204 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {1205 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);1206 dev->flags |= HIFN_FLAG_CMD_BUSY;1207 }1208 return 0;1209 1210err_out:1211 return -EINVAL;1212}1213 1214static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,1215 unsigned int offset, unsigned int size, int last)1216{1217 struct hifn_dma *dma = dev->desc_virt;1218 int idx;1219 dma_addr_t addr;1220 1221 addr = dma_map_page(&dev->pdev->dev, page, offset, size,1222 DMA_TO_DEVICE);1223 1224 idx = dma->srci;1225 1226 dma->srcr[idx].p = __cpu_to_le32(addr);1227 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |1228 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));1229 1230 if (++idx == HIFN_D_SRC_RSIZE) {1231 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |1232 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |1233 (last ? HIFN_D_LAST : 0));1234 idx = 0;1235 }1236 1237 dma->srci = idx;1238 dma->srcu++;1239 1240 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {1241 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);1242 dev->flags |= HIFN_FLAG_SRC_BUSY;1243 }1244 1245 return size;1246}1247 1248static void hifn_setup_res_desc(struct hifn_device *dev)1249{1250 struct hifn_dma *dma = dev->desc_virt;1251 1252 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |1253 HIFN_D_VALID | HIFN_D_LAST);1254 /*1255 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |1256 * HIFN_D_LAST);1257 */1258 1259 if (++dma->resi == HIFN_D_RES_RSIZE) {1260 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |1261 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);1262 dma->resi = 0;1263 }1264 1265 dma->resu++;1266 1267 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {1268 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);1269 dev->flags |= HIFN_FLAG_RES_BUSY;1270 }1271}1272 1273static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,1274 unsigned offset, unsigned size, int last)1275{1276 struct hifn_dma *dma = dev->desc_virt;1277 int idx;1278 dma_addr_t addr;1279 1280 addr = dma_map_page(&dev->pdev->dev, page, offset, size,1281 DMA_FROM_DEVICE);1282 1283 idx = dma->dsti;1284 dma->dstr[idx].p = __cpu_to_le32(addr);1285 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |1286 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));1287 1288 if (++idx == HIFN_D_DST_RSIZE) {1289 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |1290 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |1291 (last ? HIFN_D_LAST : 0));1292 idx = 0;1293 }1294 dma->dsti = idx;1295 dma->dstu++;1296 1297 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {1298 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);1299 dev->flags |= HIFN_FLAG_DST_BUSY;1300 }1301}1302 1303static int hifn_setup_dma(struct hifn_device *dev,1304 struct hifn_context *ctx, struct hifn_request_context *rctx,1305 struct scatterlist *src, struct scatterlist *dst,1306 unsigned int nbytes, void *priv)1307{1308 struct scatterlist *t;1309 struct page *spage, *dpage;1310 unsigned int soff, doff;1311 unsigned int n, len;1312 1313 n = nbytes;1314 while (n) {1315 spage = sg_page(src);1316 soff = src->offset;1317 len = min(src->length, n);1318 1319 hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);1320 1321 src++;1322 n -= len;1323 }1324 1325 t = &rctx->walk.cache[0];1326 n = nbytes;1327 while (n) {1328 if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {1329 BUG_ON(!sg_page(t));1330 dpage = sg_page(t);1331 doff = 0;1332 len = t->length;1333 } else {1334 BUG_ON(!sg_page(dst));1335 dpage = sg_page(dst);1336 doff = dst->offset;1337 len = dst->length;1338 }1339 len = min(len, n);1340 1341 hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);1342 1343 dst++;1344 t++;1345 n -= len;1346 }1347 1348 hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);1349 hifn_setup_res_desc(dev);1350 return 0;1351}1352 1353static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,1354 int num, gfp_t gfp_flags)1355{1356 int i;1357 1358 num = min(ASYNC_SCATTERLIST_CACHE, num);1359 sg_init_table(w->cache, num);1360 1361 w->num = 0;1362 for (i = 0; i < num; ++i) {1363 struct page *page = alloc_page(gfp_flags);1364 struct scatterlist *s;1365 1366 if (!page)1367 break;1368 1369 s = &w->cache[i];1370 1371 sg_set_page(s, page, PAGE_SIZE, 0);1372 w->num++;1373 }1374 1375 return i;1376}1377 1378static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)1379{1380 int i;1381 1382 for (i = 0; i < w->num; ++i) {1383 struct scatterlist *s = &w->cache[i];1384 1385 __free_page(sg_page(s));1386 1387 s->length = 0;1388 }1389 1390 w->num = 0;1391}1392 1393static int skcipher_add(unsigned int *drestp, struct scatterlist *dst,1394 unsigned int size, unsigned int *nbytesp)1395{1396 unsigned int copy, drest = *drestp, nbytes = *nbytesp;1397 int idx = 0;1398 1399 if (drest < size || size > nbytes)1400 return -EINVAL;1401 1402 while (size) {1403 copy = min3(drest, size, dst->length);1404 1405 size -= copy;1406 drest -= copy;1407 nbytes -= copy;1408 1409 pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",1410 __func__, copy, size, drest, nbytes);1411 1412 dst++;1413 idx++;1414 }1415 1416 *nbytesp = nbytes;1417 *drestp = drest;1418 1419 return idx;1420}1421 1422static int hifn_cipher_walk(struct skcipher_request *req,1423 struct hifn_cipher_walk *w)1424{1425 struct scatterlist *dst, *t;1426 unsigned int nbytes = req->cryptlen, offset, copy, diff;1427 int idx, tidx, err;1428 1429 tidx = idx = 0;1430 offset = 0;1431 while (nbytes) {1432 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))1433 return -EINVAL;1434 1435 dst = &req->dst[idx];1436 1437 pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",1438 __func__, dst->length, dst->offset, offset, nbytes);1439 1440 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||1441 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||1442 offset) {1443 unsigned slen = min(dst->length - offset, nbytes);1444 unsigned dlen = PAGE_SIZE;1445 1446 t = &w->cache[idx];1447 1448 err = skcipher_add(&dlen, dst, slen, &nbytes);1449 if (err < 0)1450 return err;1451 1452 idx += err;1453 1454 copy = slen & ~(HIFN_D_DST_DALIGN - 1);1455 diff = slen & (HIFN_D_DST_DALIGN - 1);1456 1457 if (dlen < nbytes) {1458 /*1459 * Destination page does not have enough space1460 * to put there additional blocksized chunk,1461 * so we mark that page as containing only1462 * blocksize aligned chunks:1463 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));1464 * and increase number of bytes to be processed1465 * in next chunk:1466 * nbytes += diff;1467 */1468 nbytes += diff;1469 1470 /*1471 * Temporary of course...1472 * Kick author if you will catch this one.1473 */1474 pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n",1475 __func__, dlen, nbytes, slen, offset);1476 pr_err("%s: please contact author to fix this "1477 "issue, generally you should not catch "1478 "this path under any condition but who "1479 "knows how did you use crypto code.\n"1480 "Thank you.\n", __func__);1481 BUG();1482 } else {1483 copy += diff + nbytes;1484 1485 dst = &req->dst[idx];1486 1487 err = skcipher_add(&dlen, dst, nbytes, &nbytes);1488 if (err < 0)1489 return err;1490 1491 idx += err;1492 }1493 1494 t->length = copy;1495 t->offset = offset;1496 } else {1497 nbytes -= min(dst->length, nbytes);1498 idx++;1499 }1500 1501 tidx++;1502 }1503 1504 return tidx;1505}1506 1507static int hifn_setup_session(struct skcipher_request *req)1508{1509 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);1510 struct hifn_request_context *rctx = skcipher_request_ctx(req);1511 struct hifn_device *dev = ctx->dev;1512 unsigned long dlen, flags;1513 unsigned int nbytes = req->cryptlen, idx = 0;1514 int err = -EINVAL, sg_num;1515 struct scatterlist *dst;1516 1517 if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)1518 goto err_out_exit;1519 1520 rctx->walk.flags = 0;1521 1522 while (nbytes) {1523 dst = &req->dst[idx];1524 dlen = min(dst->length, nbytes);1525 1526 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||1527 !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))1528 rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;1529 1530 nbytes -= dlen;1531 idx++;1532 }1533 1534 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {1535 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);1536 if (err < 0)1537 return err;1538 }1539 1540 sg_num = hifn_cipher_walk(req, &rctx->walk);1541 if (sg_num < 0) {1542 err = sg_num;1543 goto err_out_exit;1544 }1545 1546 spin_lock_irqsave(&dev->lock, flags);1547 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {1548 err = -EAGAIN;1549 goto err_out;1550 }1551 1552 err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req);1553 if (err)1554 goto err_out;1555 1556 dev->snum++;1557 1558 dev->active = HIFN_DEFAULT_ACTIVE_NUM;1559 spin_unlock_irqrestore(&dev->lock, flags);1560 1561 return 0;1562 1563err_out:1564 spin_unlock_irqrestore(&dev->lock, flags);1565err_out_exit:1566 if (err) {1567 dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, "1568 "type: %u, err: %d.\n",1569 rctx->iv, rctx->ivsize,1570 ctx->key, ctx->keysize,1571 rctx->mode, rctx->op, rctx->type, err);1572 }1573 1574 return err;1575}1576 1577static int hifn_start_device(struct hifn_device *dev)1578{1579 int err;1580 1581 dev->started = dev->active = 0;1582 hifn_reset_dma(dev, 1);1583 1584 err = hifn_enable_crypto(dev);1585 if (err)1586 return err;1587 1588 hifn_reset_puc(dev);1589 1590 hifn_init_dma(dev);1591 1592 hifn_init_registers(dev);1593 1594 hifn_init_pubrng(dev);1595 1596 return 0;1597}1598 1599static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,1600 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)1601{1602 unsigned int srest = *srestp, nbytes = *nbytesp, copy;1603 void *daddr;1604 int idx = 0;1605 1606 if (srest < size || size > nbytes)1607 return -EINVAL;1608 1609 while (size) {1610 copy = min3(srest, dst->length, size);1611 1612 daddr = kmap_atomic(sg_page(dst));1613 memcpy(daddr + dst->offset + offset, saddr, copy);1614 kunmap_atomic(daddr);1615 1616 nbytes -= copy;1617 size -= copy;1618 srest -= copy;1619 saddr += copy;1620 offset = 0;1621 1622 pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",1623 __func__, copy, size, srest, nbytes);1624 1625 dst++;1626 idx++;1627 }1628 1629 *nbytesp = nbytes;1630 *srestp = srest;1631 1632 return idx;1633}1634 1635static inline void hifn_complete_sa(struct hifn_device *dev, int i)1636{1637 unsigned long flags;1638 1639 spin_lock_irqsave(&dev->lock, flags);1640 dev->sa[i] = NULL;1641 dev->started--;1642 if (dev->started < 0)1643 dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__,1644 dev->started);1645 spin_unlock_irqrestore(&dev->lock, flags);1646 BUG_ON(dev->started < 0);1647}1648 1649static void hifn_process_ready(struct skcipher_request *req, int error)1650{1651 struct hifn_request_context *rctx = skcipher_request_ctx(req);1652 1653 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {1654 unsigned int nbytes = req->cryptlen;1655 int idx = 0, err;1656 struct scatterlist *dst, *t;1657 void *saddr;1658 1659 while (nbytes) {1660 t = &rctx->walk.cache[idx];1661 dst = &req->dst[idx];1662 1663 pr_debug("\n%s: sg_page(t): %p, t->length: %u, "1664 "sg_page(dst): %p, dst->length: %u, "1665 "nbytes: %u.\n",1666 __func__, sg_page(t), t->length,1667 sg_page(dst), dst->length, nbytes);1668 1669 if (!t->length) {1670 nbytes -= min(dst->length, nbytes);1671 idx++;1672 continue;1673 }1674 1675 saddr = kmap_atomic(sg_page(t));1676 1677 err = skcipher_get(saddr, &t->length, t->offset,1678 dst, nbytes, &nbytes);1679 if (err < 0) {1680 kunmap_atomic(saddr);1681 break;1682 }1683 1684 idx += err;1685 kunmap_atomic(saddr);1686 }1687 1688 hifn_cipher_walk_exit(&rctx->walk);1689 }1690 1691 skcipher_request_complete(req, error);1692}1693 1694static void hifn_clear_rings(struct hifn_device *dev, int error)1695{1696 struct hifn_dma *dma = dev->desc_virt;1697 int i, u;1698 1699 dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "1700 "k: %d.%d.%d.%d.\n",1701 dma->cmdi, dma->srci, dma->dsti, dma->resi,1702 dma->cmdu, dma->srcu, dma->dstu, dma->resu,1703 dma->cmdk, dma->srck, dma->dstk, dma->resk);1704 1705 i = dma->resk; u = dma->resu;1706 while (u != 0) {1707 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))1708 break;1709 1710 if (dev->sa[i]) {1711 dev->success++;1712 dev->reset = 0;1713 hifn_process_ready(dev->sa[i], error);1714 hifn_complete_sa(dev, i);1715 }1716 1717 if (++i == HIFN_D_RES_RSIZE)1718 i = 0;1719 u--;1720 }1721 dma->resk = i; dma->resu = u;1722 1723 i = dma->srck; u = dma->srcu;1724 while (u != 0) {1725 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))1726 break;1727 if (++i == HIFN_D_SRC_RSIZE)1728 i = 0;1729 u--;1730 }1731 dma->srck = i; dma->srcu = u;1732 1733 i = dma->cmdk; u = dma->cmdu;1734 while (u != 0) {1735 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))1736 break;1737 if (++i == HIFN_D_CMD_RSIZE)1738 i = 0;1739 u--;1740 }1741 dma->cmdk = i; dma->cmdu = u;1742 1743 i = dma->dstk; u = dma->dstu;1744 while (u != 0) {1745 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))1746 break;1747 if (++i == HIFN_D_DST_RSIZE)1748 i = 0;1749 u--;1750 }1751 dma->dstk = i; dma->dstu = u;1752 1753 dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "1754 "k: %d.%d.%d.%d.\n",1755 dma->cmdi, dma->srci, dma->dsti, dma->resi,1756 dma->cmdu, dma->srcu, dma->dstu, dma->resu,1757 dma->cmdk, dma->srck, dma->dstk, dma->resk);1758}1759 1760static void hifn_work(struct work_struct *work)1761{1762 struct delayed_work *dw = to_delayed_work(work);1763 struct hifn_device *dev = container_of(dw, struct hifn_device, work);1764 unsigned long flags;1765 int reset = 0;1766 u32 r = 0;1767 1768 spin_lock_irqsave(&dev->lock, flags);1769 if (dev->active == 0) {1770 struct hifn_dma *dma = dev->desc_virt;1771 1772 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {1773 dev->flags &= ~HIFN_FLAG_CMD_BUSY;1774 r |= HIFN_DMACSR_C_CTRL_DIS;1775 }1776 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {1777 dev->flags &= ~HIFN_FLAG_SRC_BUSY;1778 r |= HIFN_DMACSR_S_CTRL_DIS;1779 }1780 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {1781 dev->flags &= ~HIFN_FLAG_DST_BUSY;1782 r |= HIFN_DMACSR_D_CTRL_DIS;1783 }1784 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {1785 dev->flags &= ~HIFN_FLAG_RES_BUSY;1786 r |= HIFN_DMACSR_R_CTRL_DIS;1787 }1788 if (r)1789 hifn_write_1(dev, HIFN_1_DMA_CSR, r);1790 } else1791 dev->active--;1792 1793 if ((dev->prev_success == dev->success) && dev->started)1794 reset = 1;1795 dev->prev_success = dev->success;1796 spin_unlock_irqrestore(&dev->lock, flags);1797 1798 if (reset) {1799 if (++dev->reset >= 5) {1800 int i;1801 struct hifn_dma *dma = dev->desc_virt;1802 1803 dev_info(&dev->pdev->dev,1804 "r: %08x, active: %d, started: %d, "1805 "success: %lu: qlen: %u/%u, reset: %d.\n",1806 r, dev->active, dev->started,1807 dev->success, dev->queue.qlen, dev->queue.max_qlen,1808 reset);1809 1810 dev_info(&dev->pdev->dev, "%s: res: ", __func__);1811 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {1812 pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]);1813 if (dev->sa[i]) {1814 hifn_process_ready(dev->sa[i], -ENODEV);1815 hifn_complete_sa(dev, i);1816 }1817 }1818 pr_info("\n");1819 1820 hifn_reset_dma(dev, 1);1821 hifn_stop_device(dev);1822 hifn_start_device(dev);1823 dev->reset = 0;1824 }1825 1826 tasklet_schedule(&dev->tasklet);1827 }1828 1829 schedule_delayed_work(&dev->work, HZ);1830}1831 1832static irqreturn_t hifn_interrupt(int irq, void *data)1833{1834 struct hifn_device *dev = data;1835 struct hifn_dma *dma = dev->desc_virt;1836 u32 dmacsr, restart;1837 1838 dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);1839 1840 dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "1841 "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",1842 dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,1843 dma->cmdi, dma->srci, dma->dsti, dma->resi,1844 dma->cmdu, dma->srcu, dma->dstu, dma->resu);1845 1846 if ((dmacsr & dev->dmareg) == 0)1847 return IRQ_NONE;1848 1849 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);1850 1851 if (dmacsr & HIFN_DMACSR_ENGINE)1852 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));1853 if (dmacsr & HIFN_DMACSR_PUBDONE)1854 hifn_write_1(dev, HIFN_1_PUB_STATUS,1855 hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);1856 1857 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);1858 if (restart) {1859 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);1860 1861 dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",1862 !!(dmacsr & HIFN_DMACSR_R_OVER),1863 !!(dmacsr & HIFN_DMACSR_D_OVER),1864 puisr, !!(puisr & HIFN_PUISR_DSTOVER));1865 if (!!(puisr & HIFN_PUISR_DSTOVER))1866 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);1867 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |1868 HIFN_DMACSR_D_OVER));1869 }1870 1871 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |1872 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);1873 if (restart) {1874 dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n",1875 !!(dmacsr & HIFN_DMACSR_C_ABORT),1876 !!(dmacsr & HIFN_DMACSR_S_ABORT),1877 !!(dmacsr & HIFN_DMACSR_D_ABORT),1878 !!(dmacsr & HIFN_DMACSR_R_ABORT));1879 hifn_reset_dma(dev, 1);1880 hifn_init_dma(dev);1881 hifn_init_registers(dev);1882 }1883 1884 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {1885 dev_dbg(&dev->pdev->dev, "wait on command.\n");1886 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);1887 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);1888 }1889 1890 tasklet_schedule(&dev->tasklet);1891 1892 return IRQ_HANDLED;1893}1894 1895static void hifn_flush(struct hifn_device *dev)1896{1897 unsigned long flags;1898 struct crypto_async_request *async_req;1899 struct skcipher_request *req;1900 struct hifn_dma *dma = dev->desc_virt;1901 int i;1902 1903 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {1904 struct hifn_desc *d = &dma->resr[i];1905 1906 if (dev->sa[i]) {1907 hifn_process_ready(dev->sa[i],1908 (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0);1909 hifn_complete_sa(dev, i);1910 }1911 }1912 1913 spin_lock_irqsave(&dev->lock, flags);1914 while ((async_req = crypto_dequeue_request(&dev->queue))) {1915 req = skcipher_request_cast(async_req);1916 spin_unlock_irqrestore(&dev->lock, flags);1917 1918 hifn_process_ready(req, -ENODEV);1919 1920 spin_lock_irqsave(&dev->lock, flags);1921 }1922 spin_unlock_irqrestore(&dev->lock, flags);1923}1924 1925static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key,1926 unsigned int len)1927{1928 struct hifn_context *ctx = crypto_skcipher_ctx(cipher);1929 struct hifn_device *dev = ctx->dev;1930 int err;1931 1932 err = verify_skcipher_des_key(cipher, key);1933 if (err)1934 return err;1935 1936 dev->flags &= ~HIFN_FLAG_OLD_KEY;1937 1938 memcpy(ctx->key, key, len);1939 ctx->keysize = len;1940 1941 return 0;1942}1943 1944static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,1945 unsigned int len)1946{1947 struct hifn_context *ctx = crypto_skcipher_ctx(cipher);1948 struct hifn_device *dev = ctx->dev;1949 int err;1950 1951 err = verify_skcipher_des3_key(cipher, key);1952 if (err)1953 return err;1954 1955 dev->flags &= ~HIFN_FLAG_OLD_KEY;1956 1957 memcpy(ctx->key, key, len);1958 ctx->keysize = len;1959 1960 return 0;1961}1962 1963static int hifn_handle_req(struct skcipher_request *req)1964{1965 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);1966 struct hifn_device *dev = ctx->dev;1967 int err = -EAGAIN;1968 1969 if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)1970 err = hifn_setup_session(req);1971 1972 if (err == -EAGAIN) {1973 unsigned long flags;1974 1975 spin_lock_irqsave(&dev->lock, flags);1976 err = crypto_enqueue_request(&dev->queue, &req->base);1977 spin_unlock_irqrestore(&dev->lock, flags);1978 }1979 1980 return err;1981}1982 1983static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op,1984 u8 type, u8 mode)1985{1986 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);1987 struct hifn_request_context *rctx = skcipher_request_ctx(req);1988 unsigned ivsize;1989 1990 ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));1991 1992 if (req->iv && mode != ACRYPTO_MODE_ECB) {1993 if (type == ACRYPTO_TYPE_AES_128)1994 ivsize = HIFN_AES_IV_LENGTH;1995 else if (type == ACRYPTO_TYPE_DES)1996 ivsize = HIFN_DES_KEY_LENGTH;1997 else if (type == ACRYPTO_TYPE_3DES)1998 ivsize = HIFN_3DES_KEY_LENGTH;1999 }2000 2001 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {2002 if (ctx->keysize == 24)2003 type = ACRYPTO_TYPE_AES_192;2004 else if (ctx->keysize == 32)2005 type = ACRYPTO_TYPE_AES_256;2006 }2007 2008 rctx->op = op;2009 rctx->mode = mode;2010 rctx->type = type;2011 rctx->iv = req->iv;2012 rctx->ivsize = ivsize;2013 2014 /*2015 * HEAVY TODO: needs to kick Herbert XU to write documentation.2016 * HEAVY TODO: needs to kick Herbert XU to write documentation.2017 * HEAVY TODO: needs to kick Herbert XU to write documentation.2018 */2019 2020 return hifn_handle_req(req);2021}2022 2023static int hifn_process_queue(struct hifn_device *dev)2024{2025 struct crypto_async_request *async_req, *backlog;2026 struct skcipher_request *req;2027 unsigned long flags;2028 int err = 0;2029 2030 while (dev->started < HIFN_QUEUE_LENGTH) {2031 spin_lock_irqsave(&dev->lock, flags);2032 backlog = crypto_get_backlog(&dev->queue);2033 async_req = crypto_dequeue_request(&dev->queue);2034 spin_unlock_irqrestore(&dev->lock, flags);2035 2036 if (!async_req)2037 break;2038 2039 if (backlog)2040 crypto_request_complete(backlog, -EINPROGRESS);2041 2042 req = skcipher_request_cast(async_req);2043 2044 err = hifn_handle_req(req);2045 if (err)2046 break;2047 }2048 2049 return err;2050}2051 2052static int hifn_setup_crypto(struct skcipher_request *req, u8 op,2053 u8 type, u8 mode)2054{2055 int err;2056 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);2057 struct hifn_device *dev = ctx->dev;2058 2059 err = hifn_setup_crypto_req(req, op, type, mode);2060 if (err)2061 return err;2062 2063 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)2064 hifn_process_queue(dev);2065 2066 return -EINPROGRESS;2067}2068 2069/*2070 * AES ecryption functions.2071 */2072static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req)2073{2074 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,2075 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);2076}2077static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req)2078{2079 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,2080 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);2081}2082 2083/*2084 * AES decryption functions.2085 */2086static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req)2087{2088 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,2089 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);2090}2091static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req)2092{2093 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,2094 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);2095}2096 2097/*2098 * DES ecryption functions.2099 */2100static inline int hifn_encrypt_des_ecb(struct skcipher_request *req)2101{2102 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,2103 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);2104}2105static inline int hifn_encrypt_des_cbc(struct skcipher_request *req)2106{2107 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,2108 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);2109}2110 2111/*2112 * DES decryption functions.2113 */2114static inline int hifn_decrypt_des_ecb(struct skcipher_request *req)2115{2116 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,2117 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);2118}2119static inline int hifn_decrypt_des_cbc(struct skcipher_request *req)2120{2121 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,2122 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);2123}2124 2125/*2126 * 3DES ecryption functions.2127 */2128static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req)2129{2130 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,2131 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);2132}2133static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req)2134{2135 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,2136 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);2137}2138 2139/* 3DES decryption functions. */2140static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req)2141{2142 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,2143 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);2144}2145static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req)2146{2147 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,2148 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);2149}2150 2151struct hifn_alg_template {2152 char name[CRYPTO_MAX_ALG_NAME];2153 char drv_name[CRYPTO_MAX_ALG_NAME];2154 unsigned int bsize;2155 struct skcipher_alg skcipher;2156};2157 2158static const struct hifn_alg_template hifn_alg_templates[] = {2159 /*2160 * 3DES ECB and CBC modes.2161 */2162 {2163 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,2164 .skcipher = {2165 .ivsize = HIFN_IV_LENGTH,2166 .min_keysize = HIFN_3DES_KEY_LENGTH,2167 .max_keysize = HIFN_3DES_KEY_LENGTH,2168 .setkey = hifn_des3_setkey,2169 .encrypt = hifn_encrypt_3des_cbc,2170 .decrypt = hifn_decrypt_3des_cbc,2171 },2172 },2173 {2174 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,2175 .skcipher = {2176 .min_keysize = HIFN_3DES_KEY_LENGTH,2177 .max_keysize = HIFN_3DES_KEY_LENGTH,2178 .setkey = hifn_des3_setkey,2179 .encrypt = hifn_encrypt_3des_ecb,2180 .decrypt = hifn_decrypt_3des_ecb,2181 },2182 },2183 2184 /*2185 * DES ECB and CBC modes.2186 */2187 {2188 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,2189 .skcipher = {2190 .ivsize = HIFN_IV_LENGTH,2191 .min_keysize = HIFN_DES_KEY_LENGTH,2192 .max_keysize = HIFN_DES_KEY_LENGTH,2193 .setkey = hifn_setkey,2194 .encrypt = hifn_encrypt_des_cbc,2195 .decrypt = hifn_decrypt_des_cbc,2196 },2197 },2198 {2199 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,2200 .skcipher = {2201 .min_keysize = HIFN_DES_KEY_LENGTH,2202 .max_keysize = HIFN_DES_KEY_LENGTH,2203 .setkey = hifn_setkey,2204 .encrypt = hifn_encrypt_des_ecb,2205 .decrypt = hifn_decrypt_des_ecb,2206 },2207 },2208 2209 /*2210 * AES ECB and CBC modes.2211 */2212 {2213 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,2214 .skcipher = {2215 .min_keysize = AES_MIN_KEY_SIZE,2216 .max_keysize = AES_MAX_KEY_SIZE,2217 .setkey = hifn_setkey,2218 .encrypt = hifn_encrypt_aes_ecb,2219 .decrypt = hifn_decrypt_aes_ecb,2220 },2221 },2222 {2223 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,2224 .skcipher = {2225 .ivsize = HIFN_AES_IV_LENGTH,2226 .min_keysize = AES_MIN_KEY_SIZE,2227 .max_keysize = AES_MAX_KEY_SIZE,2228 .setkey = hifn_setkey,2229 .encrypt = hifn_encrypt_aes_cbc,2230 .decrypt = hifn_decrypt_aes_cbc,2231 },2232 },2233};2234 2235static int hifn_init_tfm(struct crypto_skcipher *tfm)2236{2237 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);2238 struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);2239 struct hifn_context *ctx = crypto_skcipher_ctx(tfm);2240 2241 ctx->dev = ha->dev;2242 crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context));2243 2244 return 0;2245}2246 2247static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t)2248{2249 struct hifn_crypto_alg *alg;2250 int err;2251 2252 alg = kzalloc(sizeof(*alg), GFP_KERNEL);2253 if (!alg)2254 return -ENOMEM;2255 2256 alg->alg = t->skcipher;2257 alg->alg.init = hifn_init_tfm;2258 2259 err = -EINVAL;2260 if (snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,2261 "%s", t->name) >= CRYPTO_MAX_ALG_NAME)2262 goto out_free_alg;2263 if (snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,2264 "%s-%s", t->drv_name, dev->name) >= CRYPTO_MAX_ALG_NAME)2265 goto out_free_alg;2266 2267 alg->alg.base.cra_priority = 300;2268 alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;2269 alg->alg.base.cra_blocksize = t->bsize;2270 alg->alg.base.cra_ctxsize = sizeof(struct hifn_context);2271 alg->alg.base.cra_alignmask = 0;2272 alg->alg.base.cra_module = THIS_MODULE;2273 2274 alg->dev = dev;2275 2276 list_add_tail(&alg->entry, &dev->alg_list);2277 2278 err = crypto_register_skcipher(&alg->alg);2279 if (err) {2280 list_del(&alg->entry);2281out_free_alg:2282 kfree(alg);2283 }2284 2285 return err;2286}2287 2288static void hifn_unregister_alg(struct hifn_device *dev)2289{2290 struct hifn_crypto_alg *a, *n;2291 2292 list_for_each_entry_safe(a, n, &dev->alg_list, entry) {2293 list_del(&a->entry);2294 crypto_unregister_skcipher(&a->alg);2295 kfree(a);2296 }2297}2298 2299static int hifn_register_alg(struct hifn_device *dev)2300{2301 int i, err;2302 2303 for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) {2304 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);2305 if (err)2306 goto err_out_exit;2307 }2308 2309 return 0;2310 2311err_out_exit:2312 hifn_unregister_alg(dev);2313 return err;2314}2315 2316static void hifn_tasklet_callback(unsigned long data)2317{2318 struct hifn_device *dev = (struct hifn_device *)data;2319 2320 /*2321 * This is ok to call this without lock being held,2322 * althogh it modifies some parameters used in parallel,2323 * (like dev->success), but they are used in process2324 * context or update is atomic (like setting dev->sa[i] to NULL).2325 */2326 hifn_clear_rings(dev, 0);2327 2328 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)2329 hifn_process_queue(dev);2330}2331 2332static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)2333{2334 int err, i;2335 struct hifn_device *dev;2336 char name[8];2337 2338 err = pci_enable_device(pdev);2339 if (err)2340 return err;2341 pci_set_master(pdev);2342 2343 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));2344 if (err)2345 goto err_out_disable_pci_device;2346 2347 snprintf(name, sizeof(name), "hifn%d",2348 atomic_inc_return(&hifn_dev_number) - 1);2349 2350 err = pci_request_regions(pdev, name);2351 if (err)2352 goto err_out_disable_pci_device;2353 2354 if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||2355 pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||2356 pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {2357 dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n");2358 err = -ENODEV;2359 goto err_out_free_regions;2360 }2361 2362 dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),2363 GFP_KERNEL);2364 if (!dev) {2365 err = -ENOMEM;2366 goto err_out_free_regions;2367 }2368 2369 INIT_LIST_HEAD(&dev->alg_list);2370 2371 snprintf(dev->name, sizeof(dev->name), "%s", name);2372 spin_lock_init(&dev->lock);2373 2374 for (i = 0; i < 3; ++i) {2375 unsigned long addr, size;2376 2377 addr = pci_resource_start(pdev, i);2378 size = pci_resource_len(pdev, i);2379 2380 dev->bar[i] = ioremap(addr, size);2381 if (!dev->bar[i]) {2382 err = -ENOMEM;2383 goto err_out_unmap_bars;2384 }2385 }2386 2387 dev->desc_virt = dma_alloc_coherent(&pdev->dev,2388 sizeof(struct hifn_dma),2389 &dev->desc_dma, GFP_KERNEL);2390 if (!dev->desc_virt) {2391 dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n");2392 err = -ENOMEM;2393 goto err_out_unmap_bars;2394 }2395 2396 dev->pdev = pdev;2397 dev->irq = pdev->irq;2398 2399 for (i = 0; i < HIFN_D_RES_RSIZE; ++i)2400 dev->sa[i] = NULL;2401 2402 pci_set_drvdata(pdev, dev);2403 2404 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);2405 2406 crypto_init_queue(&dev->queue, 1);2407 2408 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);2409 if (err) {2410 dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n",2411 dev->irq, err);2412 dev->irq = 0;2413 goto err_out_free_desc;2414 }2415 2416 err = hifn_start_device(dev);2417 if (err)2418 goto err_out_free_irq;2419 2420 err = hifn_register_rng(dev);2421 if (err)2422 goto err_out_stop_device;2423 2424 err = hifn_register_alg(dev);2425 if (err)2426 goto err_out_unregister_rng;2427 2428 INIT_DELAYED_WORK(&dev->work, hifn_work);2429 schedule_delayed_work(&dev->work, HZ);2430 2431 dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been "2432 "successfully registered as %s.\n",2433 pci_name(pdev), dev->name);2434 2435 return 0;2436 2437err_out_unregister_rng:2438 hifn_unregister_rng(dev);2439err_out_stop_device:2440 hifn_reset_dma(dev, 1);2441 hifn_stop_device(dev);2442err_out_free_irq:2443 free_irq(dev->irq, dev);2444 tasklet_kill(&dev->tasklet);2445err_out_free_desc:2446 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt,2447 dev->desc_dma);2448 2449err_out_unmap_bars:2450 for (i = 0; i < 3; ++i)2451 if (dev->bar[i])2452 iounmap(dev->bar[i]);2453 kfree(dev);2454 2455err_out_free_regions:2456 pci_release_regions(pdev);2457 2458err_out_disable_pci_device:2459 pci_disable_device(pdev);2460 2461 return err;2462}2463 2464static void hifn_remove(struct pci_dev *pdev)2465{2466 int i;2467 struct hifn_device *dev;2468 2469 dev = pci_get_drvdata(pdev);2470 2471 if (dev) {2472 cancel_delayed_work_sync(&dev->work);2473 2474 hifn_unregister_rng(dev);2475 hifn_unregister_alg(dev);2476 hifn_reset_dma(dev, 1);2477 hifn_stop_device(dev);2478 2479 free_irq(dev->irq, dev);2480 tasklet_kill(&dev->tasklet);2481 2482 hifn_flush(dev);2483 2484 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma),2485 dev->desc_virt, dev->desc_dma);2486 for (i = 0; i < 3; ++i)2487 if (dev->bar[i])2488 iounmap(dev->bar[i]);2489 2490 kfree(dev);2491 }2492 2493 pci_release_regions(pdev);2494 pci_disable_device(pdev);2495}2496 2497static struct pci_device_id hifn_pci_tbl[] = {2498 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },2499 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },2500 { 0 }2501};2502MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);2503 2504static struct pci_driver hifn_pci_driver = {2505 .name = "hifn795x",2506 .id_table = hifn_pci_tbl,2507 .probe = hifn_probe,2508 .remove = hifn_remove,2509};2510 2511static int __init hifn_init(void)2512{2513 unsigned int freq;2514 int err;2515 2516 if (strncmp(hifn_pll_ref, "ext", 3) &&2517 strncmp(hifn_pll_ref, "pci", 3)) {2518 pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext");2519 return -EINVAL;2520 }2521 2522 /*2523 * For the 7955/7956 the reference clock frequency must be in the2524 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,2525 * but this chip is currently not supported.2526 */2527 if (hifn_pll_ref[3] != '\0') {2528 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);2529 if (freq < 20 || freq > 100) {2530 pr_err("hifn795x: invalid hifn_pll_ref frequency, must"2531 "be in the range of 20-100");2532 return -EINVAL;2533 }2534 }2535 2536 err = pci_register_driver(&hifn_pci_driver);2537 if (err < 0) {2538 pr_err("Failed to register PCI driver for %s device.\n",2539 hifn_pci_driver.name);2540 return -ENODEV;2541 }2542 2543 pr_info("Driver for HIFN 795x crypto accelerator chip "2544 "has been successfully registered.\n");2545 2546 return 0;2547}2548 2549static void __exit hifn_fini(void)2550{2551 pci_unregister_driver(&hifn_pci_driver);2552 2553 pr_info("Driver for HIFN 795x crypto accelerator chip "2554 "has been successfully unregistered.\n");2555}2556 2557module_init(hifn_init);2558module_exit(hifn_fini);2559 2560MODULE_LICENSE("GPL");2561MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");2562MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");2563