729 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#ifndef __NITROX_REQ_H3#define __NITROX_REQ_H4 5#include <linux/dma-mapping.h>6#include <crypto/aes.h>7 8#include "nitrox_dev.h"9 10#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL11#define PRIO 400112 13typedef void (*sereq_completion_t)(void *req, int err);14 15/**16 * struct gphdr - General purpose Header17 * @param0: first parameter.18 * @param1: second parameter.19 * @param2: third parameter.20 * @param3: fourth parameter.21 *22 * Params tell the iv and enc/dec data offsets.23 */24struct gphdr {25 __be16 param0;26 __be16 param1;27 __be16 param2;28 __be16 param3;29};30 31/**32 * struct se_req_ctrl - SE request information.33 * @arg: Minor number of the opcode34 * @ctxc: Context control.35 * @unca: Uncertainity enabled.36 * @info: Additional information for SE cores.37 * @ctxl: Context length in bytes.38 * @uddl: User defined data length39 */40union se_req_ctrl {41 u64 value;42 struct {43 u64 raz : 22;44 u64 arg : 8;45 u64 ctxc : 2;46 u64 unca : 1;47 u64 info : 3;48 u64 unc : 8;49 u64 ctxl : 12;50 u64 uddl : 8;51 } s;52};53 54#define MAX_IV_LEN 1655 56/**57 * struct se_crypto_request - SE crypto request structure.58 * @opcode: Request opcode (enc/dec)59 * @flags: flags from crypto subsystem60 * @ctx_handle: Crypto context handle.61 * @gph: GP Header62 * @ctrl: Request Information.63 * @orh: ORH address64 * @comp: completion address65 * @src: Input sglist66 * @dst: Output sglist67 */68struct se_crypto_request {69 u8 opcode;70 gfp_t gfp;71 u32 flags;72 u64 ctx_handle;73 74 struct gphdr gph;75 union se_req_ctrl ctrl;76 u64 *orh;77 u64 *comp;78 79 struct scatterlist *src;80 struct scatterlist *dst;81};82 83/* Crypto opcodes */84#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x3385#define ENCRYPT 086#define DECRYPT 187 88/* IV from context */89#define IV_FROM_CTX 090/* IV from Input data */91#define IV_FROM_DPTR 192 93/**94 * cipher opcodes for firmware95 */96enum flexi_cipher {97 CIPHER_NULL = 0,98 CIPHER_3DES_CBC,99 CIPHER_3DES_ECB,100 CIPHER_AES_CBC,101 CIPHER_AES_ECB,102 CIPHER_AES_CFB,103 CIPHER_AES_CTR,104 CIPHER_AES_GCM,105 CIPHER_AES_XTS,106 CIPHER_AES_CCM,107 CIPHER_AES_CBC_CTS,108 CIPHER_AES_ECB_CTS,109 CIPHER_INVALID110};111 112enum flexi_auth {113 AUTH_NULL = 0,114 AUTH_MD5,115 AUTH_SHA1,116 AUTH_SHA2_SHA224,117 AUTH_SHA2_SHA256,118 AUTH_SHA2_SHA384,119 AUTH_SHA2_SHA512,120 AUTH_GMAC,121 AUTH_INVALID122};123 124/**125 * struct crypto_keys - Crypto keys126 * @key: Encryption key or KEY1 for AES-XTS127 * @iv: Encryption IV or Tweak for AES-XTS128 */129struct crypto_keys {130 union {131 u8 key[AES_MAX_KEY_SIZE];132 u8 key1[AES_MAX_KEY_SIZE];133 } u;134 u8 iv[AES_BLOCK_SIZE];135};136 137/**138 * struct auth_keys - Authentication keys139 * @ipad: IPAD or KEY2 for AES-XTS140 * @opad: OPAD or AUTH KEY if auth_input_type = 1141 */142struct auth_keys {143 union {144 u8 ipad[64];145 u8 key2[64];146 } u;147 u8 opad[64];148};149 150union fc_ctx_flags {151 __be64 f;152 u64 fu;153 struct {154#if defined(__BIG_ENDIAN_BITFIELD)155 u64 cipher_type : 4;156 u64 reserved_59 : 1;157 u64 aes_keylen : 2;158 u64 iv_source : 1;159 u64 hash_type : 4;160 u64 reserved_49_51 : 3;161 u64 auth_input_type: 1;162 u64 mac_len : 8;163 u64 reserved_0_39 : 40;164#else165 u64 reserved_0_39 : 40;166 u64 mac_len : 8;167 u64 auth_input_type: 1;168 u64 reserved_49_51 : 3;169 u64 hash_type : 4;170 u64 iv_source : 1;171 u64 aes_keylen : 2;172 u64 reserved_59 : 1;173 u64 cipher_type : 4;174#endif175 } w0;176};177/**178 * struct flexi_crypto_context - Crypto context179 * @cipher_type: Encryption cipher type180 * @aes_keylen: AES key length181 * @iv_source: Encryption IV source182 * @hash_type: Authentication type183 * @auth_input_type: Authentication input type184 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD185 * 0 - Authentication OPAD/IPAD186 * @mac_len: mac length187 * @crypto: Crypto keys188 * @auth: Authentication keys189 */190struct flexi_crypto_context {191 union fc_ctx_flags flags;192 struct crypto_keys crypto;193 struct auth_keys auth;194};195 196struct crypto_ctx_hdr {197 struct dma_pool *pool;198 dma_addr_t dma;199 void *vaddr;200};201 202struct nitrox_crypto_ctx {203 struct nitrox_device *ndev;204 union {205 u64 ctx_handle;206 struct flexi_crypto_context *fctx;207 } u;208 struct crypto_ctx_hdr *chdr;209 sereq_completion_t callback;210};211 212struct nitrox_kcrypt_request {213 struct se_crypto_request creq;214 u8 *src;215 u8 *dst;216 u8 *iv_out;217};218 219/**220 * struct nitrox_aead_rctx - AEAD request context221 * @nkreq: Base request context222 * @cryptlen: Encryption/Decryption data length223 * @assoclen: AAD length224 * @srclen: Input buffer length225 * @dstlen: Output buffer length226 * @iv: IV data227 * @ivsize: IV data length228 * @flags: AEAD req flags229 * @ctx_handle: Device context handle230 * @src: Source sglist231 * @dst: Destination sglist232 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)233 */234struct nitrox_aead_rctx {235 struct nitrox_kcrypt_request nkreq;236 unsigned int cryptlen;237 unsigned int assoclen;238 unsigned int srclen;239 unsigned int dstlen;240 u8 *iv;241 int ivsize;242 u32 flags;243 u64 ctx_handle;244 struct scatterlist *src;245 struct scatterlist *dst;246 u8 ctrl_arg;247};248 249/**250 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context251 * @base: AEAD request context252 * @src: Source sglist253 * @dst: Destination sglist254 * @assoc: AAD255 */256struct nitrox_rfc4106_rctx {257 struct nitrox_aead_rctx base;258 struct scatterlist src[3];259 struct scatterlist dst[3];260 u8 assoc[20];261};262 263/**264 * struct pkt_instr_hdr - Packet Instruction Header265 * @g: Gather used266 * When [G] is set and [GSZ] != 0, the instruction is267 * indirect gather instruction.268 * When [G] is set and [GSZ] = 0, the instruction is269 * direct gather instruction.270 * @gsz: Number of pointers in the indirect gather list271 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr272 * and adds them to the packet after the pkt_instr_hdr but before any UDD273 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]274 * when [IHI] is set.275 * @fsz: The number of front data bytes directly included in the276 * PCIe instruction.277 * @tlen: The length of the input packet in bytes, include:278 * - 16B pkt_hdr279 * - Inline context bytes if any,280 * - UDD if any,281 * - packet payload bytes282 */283union pkt_instr_hdr {284 __be64 bev;285 u64 value;286 struct {287#if defined(__BIG_ENDIAN_BITFIELD)288 u64 raz_48_63 : 16;289 u64 g : 1;290 u64 gsz : 7;291 u64 ihi : 1;292 u64 ssz : 7;293 u64 raz_30_31 : 2;294 u64 fsz : 6;295 u64 raz_16_23 : 8;296 u64 tlen : 16;297#else298 u64 tlen : 16;299 u64 raz_16_23 : 8;300 u64 fsz : 6;301 u64 raz_30_31 : 2;302 u64 ssz : 7;303 u64 ihi : 1;304 u64 gsz : 7;305 u64 g : 1;306 u64 raz_48_63 : 16;307#endif308 } s;309};310 311/**312 * struct pkt_hdr - Packet Input Header313 * @opcode: Request opcode (Major)314 * @arg: Request opcode (Minor)315 * @ctxc: Context control.316 * @unca: When set [UNC] is the uncertainty count for an input packet.317 * The hardware uses uncertainty counts to predict318 * output buffer use and avoid deadlock.319 * @info: Not used by input hardware. Available for use320 * during SE processing.321 * @destport: The expected destination port/ring/channel for the packet.322 * @unc: Uncertainty count for an input packet.323 * @grp: SE group that will process the input packet.324 * @ctxl: Context Length in 64-bit words.325 * @uddl: User-defined data (UDD) length in bytes.326 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.327 */328union pkt_hdr {329 __be64 bev[2];330 u64 value[2];331 struct {332#if defined(__BIG_ENDIAN_BITFIELD)333 u64 opcode : 8;334 u64 arg : 8;335 u64 ctxc : 2;336 u64 unca : 1;337 u64 raz_44 : 1;338 u64 info : 3;339 u64 destport : 9;340 u64 unc : 8;341 u64 raz_19_23 : 5;342 u64 grp : 3;343 u64 raz_15 : 1;344 u64 ctxl : 7;345 u64 uddl : 8;346#else347 u64 uddl : 8;348 u64 ctxl : 7;349 u64 raz_15 : 1;350 u64 grp : 3;351 u64 raz_19_23 : 5;352 u64 unc : 8;353 u64 destport : 9;354 u64 info : 3;355 u64 raz_44 : 1;356 u64 unca : 1;357 u64 ctxc : 2;358 u64 arg : 8;359 u64 opcode : 8;360#endif361 __be64 ctxp;362 } s;363};364 365/**366 * struct slc_store_info - Solicited Paceket Output Store Information.367 * @ssz: The number of scatterlist pointers for the solicited output port368 * packet.369 * @rptr: The result pointer for the solicited output port packet.370 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote371 * host that is large enough to hold the entire output packet.372 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4373 * sglist components at [RPTR] on the remote host.374 */375union slc_store_info {376 __be64 bev[2];377 u64 value[2];378 struct {379#if defined(__BIG_ENDIAN_BITFIELD)380 u64 raz_39_63 : 25;381 u64 ssz : 7;382 u64 raz_0_31 : 32;383#else384 u64 raz_0_31 : 32;385 u64 ssz : 7;386 u64 raz_39_63 : 25;387#endif388 __be64 rptr;389 } s;390};391 392/**393 * struct nps_pkt_instr - NPS Packet Instruction of SE cores.394 * @dptr0 : Input pointer points to buffer in remote host.395 * @ih: Packet Instruction Header (8 bytes)396 * @irh: Packet Input Header (16 bytes)397 * @slc: Solicited Packet Output Store Information (16 bytes)398 * @fdata: Front data399 *400 * 64-Byte Instruction Format401 */402struct nps_pkt_instr {403 __be64 dptr0;404 union pkt_instr_hdr ih;405 union pkt_hdr irh;406 union slc_store_info slc;407 u64 fdata[2];408};409 410/**411 * struct aqmq_command_s - The 32 byte command for AE processing.412 * @opcode: Request opcode413 * @param1: Request control parameter 1414 * @param2: Request control parameter 2415 * @dlen: Input length416 * @dptr: Input pointer points to buffer in remote host417 * @rptr: Result pointer points to buffer in remote host418 * @grp: AQM Group (0..7)419 * @cptr: Context pointer420 */421struct aqmq_command_s {422 __be16 opcode;423 __be16 param1;424 __be16 param2;425 __be16 dlen;426 __be64 dptr;427 __be64 rptr;428 union {429 __be64 word3;430#if defined(__BIG_ENDIAN_BITFIELD)431 u64 grp : 3;432 u64 cptr : 61;433#else434 u64 cptr : 61;435 u64 grp : 3;436#endif437 };438};439 440/**441 * struct ctx_hdr - Book keeping data about the crypto context442 * @pool: Pool used to allocate crypto context443 * @dma: Base DMA address of the crypto context444 * @ctx_dma: Actual usable crypto context for NITROX445 */446struct ctx_hdr {447 struct dma_pool *pool;448 dma_addr_t dma;449 dma_addr_t ctx_dma;450};451 452/*453 * struct sglist_component - SG list component format454 * @len0: The number of bytes at [PTR0] on the remote host.455 * @len1: The number of bytes at [PTR1] on the remote host.456 * @len2: The number of bytes at [PTR2] on the remote host.457 * @len3: The number of bytes at [PTR3] on the remote host.458 * @dma0: First pointer point to buffer in remote host.459 * @dma1: Second pointer point to buffer in remote host.460 * @dma2: Third pointer point to buffer in remote host.461 * @dma3: Fourth pointer point to buffer in remote host.462 */463struct nitrox_sgcomp {464 __be16 len[4];465 __be64 dma[4];466};467 468/*469 * strutct nitrox_sgtable - SG list information470 * @sgmap_cnt: Number of buffers mapped471 * @total_bytes: Total bytes in sglist.472 * @sgcomp_len: Total sglist components length.473 * @sgcomp_dma: DMA address of sglist component.474 * @sg: crypto request buffer.475 * @sgcomp: sglist component for NITROX.476 */477struct nitrox_sgtable {478 u8 sgmap_cnt;479 u16 total_bytes;480 u32 sgcomp_len;481 dma_addr_t sgcomp_dma;482 struct scatterlist *sg;483 struct nitrox_sgcomp *sgcomp;484};485 486/* Response Header Length */487#define ORH_HLEN 8488/* Completion bytes Length */489#define COMP_HLEN 8490 491struct resp_hdr {492 u64 *orh;493 u64 *completion;494};495 496typedef void (*completion_t)(void *arg, int err);497 498/**499 * struct nitrox_softreq - Represents the NIROX Request.500 * @response: response list entry501 * @backlog: Backlog list entry502 * @ndev: Device used to submit the request503 * @cmdq: Command queue for submission504 * @resp: Response headers505 * @instr: 64B instruction506 * @in: SG table for input507 * @out SG table for output508 * @tstamp: Request submitted time in jiffies509 * @callback: callback after request completion/timeout510 * @cb_arg: callback argument511 */512struct nitrox_softreq {513 struct list_head response;514 struct list_head backlog;515 516 u32 flags;517 gfp_t gfp;518 atomic_t status;519 520 struct nitrox_device *ndev;521 struct nitrox_cmdq *cmdq;522 523 struct nps_pkt_instr instr;524 struct resp_hdr resp;525 struct nitrox_sgtable in;526 struct nitrox_sgtable out;527 528 unsigned long tstamp;529 530 completion_t callback;531 void *cb_arg;532};533 534static inline int flexi_aes_keylen(int keylen)535{536 int aes_keylen;537 538 switch (keylen) {539 case AES_KEYSIZE_128:540 aes_keylen = 1;541 break;542 case AES_KEYSIZE_192:543 aes_keylen = 2;544 break;545 case AES_KEYSIZE_256:546 aes_keylen = 3;547 break;548 default:549 aes_keylen = -EINVAL;550 break;551 }552 return aes_keylen;553}554 555static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)556{557 size_t size;558 559 size = sizeof(struct scatterlist) * nents;560 size += extralen;561 562 return kzalloc(size, gfp);563}564 565/**566 * create_single_sg - Point SG entry to the data567 * @sg: Destination SG list568 * @buf: Data569 * @buflen: Data length570 *571 * Returns next free entry in the destination SG list572 **/573static inline struct scatterlist *create_single_sg(struct scatterlist *sg,574 void *buf, int buflen)575{576 sg_set_buf(sg, buf, buflen);577 sg++;578 return sg;579}580 581/**582 * create_multi_sg - Create multiple sg entries with buflen data length from583 * source sglist584 * @to_sg: Destination SG list585 * @from_sg: Source SG list586 * @buflen: Data length587 *588 * Returns next free entry in the destination SG list589 **/590static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,591 struct scatterlist *from_sg,592 int buflen)593{594 struct scatterlist *sg = to_sg;595 unsigned int sglen;596 597 for (; buflen && from_sg; buflen -= sglen) {598 sglen = from_sg->length;599 if (sglen > buflen)600 sglen = buflen;601 602 sg_set_buf(sg, sg_virt(from_sg), sglen);603 from_sg = sg_next(from_sg);604 sg++;605 }606 607 return sg;608}609 610static inline void set_orh_value(u64 *orh)611{612 WRITE_ONCE(*orh, PENDING_SIG);613}614 615static inline void set_comp_value(u64 *comp)616{617 WRITE_ONCE(*comp, PENDING_SIG);618}619 620static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,621 int nents, int ivsize)622{623 struct se_crypto_request *creq = &nkreq->creq;624 625 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);626 if (!nkreq->src)627 return -ENOMEM;628 629 return 0;630}631 632static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)633{634 memcpy(dst, src, size);635}636 637static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)638{639 return (struct scatterlist *)(iv + ivsize);640}641 642static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,643 int nents, int ivsize,644 struct scatterlist *src, int buflen)645{646 char *iv = nkreq->src;647 struct scatterlist *sg;648 struct se_crypto_request *creq = &nkreq->creq;649 650 creq->src = nitrox_creq_src_sg(iv, ivsize);651 sg = creq->src;652 sg_init_table(sg, nents);653 654 /* Input format:655 * +----+----------------+656 * | IV | SRC sg entries |657 * +----+----------------+658 */659 660 /* IV */661 sg = create_single_sg(sg, iv, ivsize);662 /* SRC entries */663 create_multi_sg(sg, src, buflen);664}665 666static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,667 int nents)668{669 int extralen = ORH_HLEN + COMP_HLEN;670 struct se_crypto_request *creq = &nkreq->creq;671 672 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);673 if (!nkreq->dst)674 return -ENOMEM;675 676 return 0;677}678 679static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)680{681 struct se_crypto_request *creq = &nkreq->creq;682 683 creq->orh = (u64 *)(nkreq->dst);684 set_orh_value(creq->orh);685}686 687static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)688{689 struct se_crypto_request *creq = &nkreq->creq;690 691 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);692 set_comp_value(creq->comp);693}694 695static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)696{697 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);698}699 700static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,701 int nents, int ivsize,702 struct scatterlist *dst, int buflen)703{704 struct se_crypto_request *creq = &nkreq->creq;705 struct scatterlist *sg;706 char *iv = nkreq->src;707 708 creq->dst = nitrox_creq_dst_sg(nkreq->dst);709 sg = creq->dst;710 sg_init_table(sg, nents);711 712 /* Output format:713 * +-----+----+----------------+-----------------+714 * | ORH | IV | DST sg entries | COMPLETION Bytes|715 * +-----+----+----------------+-----------------+716 */717 718 /* ORH */719 sg = create_single_sg(sg, creq->orh, ORH_HLEN);720 /* IV */721 sg = create_single_sg(sg, iv, ivsize);722 /* DST entries */723 sg = create_multi_sg(sg, dst, buflen);724 /* COMPLETION Bytes */725 create_single_sg(sg, creq->comp, COMP_HLEN);726}727 728#endif /* __NITROX_REQ_H */729