1215 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2016 Broadcom4 */5 6#include <linux/kernel.h>7#include <linux/string.h>8 9#include "util.h"10#include "spu.h"11#include "spum.h"12#include "cipher.h"13 14char *hash_alg_name[] = { "None", "md5", "sha1", "sha224", "sha256", "aes",15 "sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", "sha3_512" };16 17char *aead_alg_name[] = { "ccm(aes)", "gcm(aes)", "authenc" };18 19/* Assumes SPU-M messages are in big endian */20void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len)21{22 u8 *ptr = buf;23 struct SPUHEADER *spuh = (struct SPUHEADER *)buf;24 unsigned int hash_key_len = 0;25 unsigned int hash_state_len = 0;26 unsigned int cipher_key_len = 0;27 unsigned int iv_len;28 u32 pflags;29 u32 cflags;30 u32 ecf;31 u32 cipher_alg;32 u32 cipher_mode;33 u32 cipher_type;34 u32 hash_alg;35 u32 hash_mode;36 u32 hash_type;37 u32 sctx_size; /* SCTX length in words */38 u32 sctx_pl_len; /* SCTX payload length in bytes */39 40 packet_log("\n");41 packet_log("SPU Message header %p len: %u\n", buf, buf_len);42 43 /* ========== Decode MH ========== */44 packet_log(" MH 0x%08x\n", be32_to_cpup((__be32 *)ptr));45 if (spuh->mh.flags & MH_SCTX_PRES)46 packet_log(" SCTX present\n");47 if (spuh->mh.flags & MH_BDESC_PRES)48 packet_log(" BDESC present\n");49 if (spuh->mh.flags & MH_MFM_PRES)50 packet_log(" MFM present\n");51 if (spuh->mh.flags & MH_BD_PRES)52 packet_log(" BD present\n");53 if (spuh->mh.flags & MH_HASH_PRES)54 packet_log(" HASH present\n");55 if (spuh->mh.flags & MH_SUPDT_PRES)56 packet_log(" SUPDT present\n");57 packet_log(" Opcode 0x%02x\n", spuh->mh.op_code);58 59 ptr += sizeof(spuh->mh) + sizeof(spuh->emh); /* skip emh. unused */60 61 /* ========== Decode SCTX ========== */62 if (spuh->mh.flags & MH_SCTX_PRES) {63 pflags = be32_to_cpu(spuh->sa.proto_flags);64 packet_log(" SCTX[0] 0x%08x\n", pflags);65 sctx_size = pflags & SCTX_SIZE;66 packet_log(" Size %u words\n", sctx_size);67 68 cflags = be32_to_cpu(spuh->sa.cipher_flags);69 packet_log(" SCTX[1] 0x%08x\n", cflags);70 packet_log(" Inbound:%lu (1:decrypt/vrfy 0:encrypt/auth)\n",71 (cflags & CIPHER_INBOUND) >> CIPHER_INBOUND_SHIFT);72 packet_log(" Order:%lu (1:AuthFirst 0:EncFirst)\n",73 (cflags & CIPHER_ORDER) >> CIPHER_ORDER_SHIFT);74 packet_log(" ICV_IS_512:%lx\n",75 (cflags & ICV_IS_512) >> ICV_IS_512_SHIFT);76 cipher_alg = (cflags & CIPHER_ALG) >> CIPHER_ALG_SHIFT;77 cipher_mode = (cflags & CIPHER_MODE) >> CIPHER_MODE_SHIFT;78 cipher_type = (cflags & CIPHER_TYPE) >> CIPHER_TYPE_SHIFT;79 packet_log(" Crypto Alg:%u Mode:%u Type:%u\n",80 cipher_alg, cipher_mode, cipher_type);81 hash_alg = (cflags & HASH_ALG) >> HASH_ALG_SHIFT;82 hash_mode = (cflags & HASH_MODE) >> HASH_MODE_SHIFT;83 hash_type = (cflags & HASH_TYPE) >> HASH_TYPE_SHIFT;84 packet_log(" Hash Alg:%x Mode:%x Type:%x\n",85 hash_alg, hash_mode, hash_type);86 packet_log(" UPDT_Offset:%u\n", cflags & UPDT_OFST);87 88 ecf = be32_to_cpu(spuh->sa.ecf);89 packet_log(" SCTX[2] 0x%08x\n", ecf);90 packet_log(" WriteICV:%lu CheckICV:%lu ICV_SIZE:%u ",91 (ecf & INSERT_ICV) >> INSERT_ICV_SHIFT,92 (ecf & CHECK_ICV) >> CHECK_ICV_SHIFT,93 (ecf & ICV_SIZE) >> ICV_SIZE_SHIFT);94 packet_log("BD_SUPPRESS:%lu\n",95 (ecf & BD_SUPPRESS) >> BD_SUPPRESS_SHIFT);96 packet_log(" SCTX_IV:%lu ExplicitIV:%lu GenIV:%lu ",97 (ecf & SCTX_IV) >> SCTX_IV_SHIFT,98 (ecf & EXPLICIT_IV) >> EXPLICIT_IV_SHIFT,99 (ecf & GEN_IV) >> GEN_IV_SHIFT);100 packet_log("IV_OV_OFST:%lu EXP_IV_SIZE:%u\n",101 (ecf & IV_OFFSET) >> IV_OFFSET_SHIFT,102 ecf & EXP_IV_SIZE);103 104 ptr += sizeof(struct SCTX);105 106 if (hash_alg && hash_mode) {107 char *name = "NONE";108 109 switch (hash_alg) {110 case HASH_ALG_MD5:111 hash_key_len = 16;112 name = "MD5";113 break;114 case HASH_ALG_SHA1:115 hash_key_len = 20;116 name = "SHA1";117 break;118 case HASH_ALG_SHA224:119 hash_key_len = 28;120 name = "SHA224";121 break;122 case HASH_ALG_SHA256:123 hash_key_len = 32;124 name = "SHA256";125 break;126 case HASH_ALG_SHA384:127 hash_key_len = 48;128 name = "SHA384";129 break;130 case HASH_ALG_SHA512:131 hash_key_len = 64;132 name = "SHA512";133 break;134 case HASH_ALG_AES:135 hash_key_len = 0;136 name = "AES";137 break;138 case HASH_ALG_NONE:139 break;140 }141 142 packet_log(" Auth Key Type:%s Length:%u Bytes\n",143 name, hash_key_len);144 packet_dump(" KEY: ", ptr, hash_key_len);145 ptr += hash_key_len;146 } else if ((hash_alg == HASH_ALG_AES) &&147 (hash_mode == HASH_MODE_XCBC)) {148 char *name = "NONE";149 150 switch (cipher_type) {151 case CIPHER_TYPE_AES128:152 hash_key_len = 16;153 name = "AES128-XCBC";154 break;155 case CIPHER_TYPE_AES192:156 hash_key_len = 24;157 name = "AES192-XCBC";158 break;159 case CIPHER_TYPE_AES256:160 hash_key_len = 32;161 name = "AES256-XCBC";162 break;163 }164 packet_log(" Auth Key Type:%s Length:%u Bytes\n",165 name, hash_key_len);166 packet_dump(" KEY: ", ptr, hash_key_len);167 ptr += hash_key_len;168 }169 170 if (hash_alg && (hash_mode == HASH_MODE_NONE) &&171 (hash_type == HASH_TYPE_UPDT)) {172 char *name = "NONE";173 174 switch (hash_alg) {175 case HASH_ALG_MD5:176 hash_state_len = 16;177 name = "MD5";178 break;179 case HASH_ALG_SHA1:180 hash_state_len = 20;181 name = "SHA1";182 break;183 case HASH_ALG_SHA224:184 hash_state_len = 32;185 name = "SHA224";186 break;187 case HASH_ALG_SHA256:188 hash_state_len = 32;189 name = "SHA256";190 break;191 case HASH_ALG_SHA384:192 hash_state_len = 48;193 name = "SHA384";194 break;195 case HASH_ALG_SHA512:196 hash_state_len = 64;197 name = "SHA512";198 break;199 case HASH_ALG_AES:200 hash_state_len = 0;201 name = "AES";202 break;203 case HASH_ALG_NONE:204 break;205 }206 207 packet_log(" Auth State Type:%s Length:%u Bytes\n",208 name, hash_state_len);209 packet_dump(" State: ", ptr, hash_state_len);210 ptr += hash_state_len;211 }212 213 if (cipher_alg) {214 char *name = "NONE";215 216 switch (cipher_alg) {217 case CIPHER_ALG_DES:218 cipher_key_len = 8;219 name = "DES";220 break;221 case CIPHER_ALG_3DES:222 cipher_key_len = 24;223 name = "3DES";224 break;225 case CIPHER_ALG_AES:226 switch (cipher_type) {227 case CIPHER_TYPE_AES128:228 cipher_key_len = 16;229 name = "AES128";230 break;231 case CIPHER_TYPE_AES192:232 cipher_key_len = 24;233 name = "AES192";234 break;235 case CIPHER_TYPE_AES256:236 cipher_key_len = 32;237 name = "AES256";238 break;239 }240 break;241 case CIPHER_ALG_NONE:242 break;243 }244 245 packet_log(" Cipher Key Type:%s Length:%u Bytes\n",246 name, cipher_key_len);247 248 /* XTS has two keys */249 if (cipher_mode == CIPHER_MODE_XTS) {250 packet_dump(" KEY2: ", ptr, cipher_key_len);251 ptr += cipher_key_len;252 packet_dump(" KEY1: ", ptr, cipher_key_len);253 ptr += cipher_key_len;254 255 cipher_key_len *= 2;256 } else {257 packet_dump(" KEY: ", ptr, cipher_key_len);258 ptr += cipher_key_len;259 }260 261 if (ecf & SCTX_IV) {262 sctx_pl_len = sctx_size * sizeof(u32) -263 sizeof(struct SCTX);264 iv_len = sctx_pl_len -265 (hash_key_len + hash_state_len +266 cipher_key_len);267 packet_log(" IV Length:%u Bytes\n", iv_len);268 packet_dump(" IV: ", ptr, iv_len);269 ptr += iv_len;270 }271 }272 }273 274 /* ========== Decode BDESC ========== */275 if (spuh->mh.flags & MH_BDESC_PRES) {276 struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr;277 278 packet_log(" BDESC[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr));279 packet_log(" OffsetMAC:%u LengthMAC:%u\n",280 be16_to_cpu(bdesc->offset_mac),281 be16_to_cpu(bdesc->length_mac));282 ptr += sizeof(u32);283 284 packet_log(" BDESC[1] 0x%08x\n", be32_to_cpup((__be32 *)ptr));285 packet_log(" OffsetCrypto:%u LengthCrypto:%u\n",286 be16_to_cpu(bdesc->offset_crypto),287 be16_to_cpu(bdesc->length_crypto));288 ptr += sizeof(u32);289 290 packet_log(" BDESC[2] 0x%08x\n", be32_to_cpup((__be32 *)ptr));291 packet_log(" OffsetICV:%u OffsetIV:%u\n",292 be16_to_cpu(bdesc->offset_icv),293 be16_to_cpu(bdesc->offset_iv));294 ptr += sizeof(u32);295 }296 297 /* ========== Decode BD ========== */298 if (spuh->mh.flags & MH_BD_PRES) {299 struct BD_HEADER *bd = (struct BD_HEADER *)ptr;300 301 packet_log(" BD[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr));302 packet_log(" Size:%ubytes PrevLength:%u\n",303 be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length));304 ptr += 4;305 }306 307 /* Double check sanity */308 if (buf + buf_len != ptr) {309 packet_log(" Packet parsed incorrectly. ");310 packet_log("buf:%p buf_len:%u buf+buf_len:%p ptr:%p\n",311 buf, buf_len, buf + buf_len, ptr);312 }313 314 packet_log("\n");315}316 317/**318 * spum_ns2_ctx_max_payload() - Determine the max length of the payload for a319 * SPU message for a given cipher and hash alg context.320 * @cipher_alg: The cipher algorithm321 * @cipher_mode: The cipher mode322 * @blocksize: The size of a block of data for this algo323 *324 * The max payload must be a multiple of the blocksize so that if a request is325 * too large to fit in a single SPU message, the request can be broken into326 * max_payload sized chunks. Each chunk must be a multiple of blocksize.327 *328 * Return: Max payload length in bytes329 */330u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg,331 enum spu_cipher_mode cipher_mode,332 unsigned int blocksize)333{334 u32 max_payload = SPUM_NS2_MAX_PAYLOAD;335 u32 excess;336 337 /* In XTS on SPU-M, we'll need to insert tweak before input data */338 if (cipher_mode == CIPHER_MODE_XTS)339 max_payload -= SPU_XTS_TWEAK_SIZE;340 341 excess = max_payload % blocksize;342 343 return max_payload - excess;344}345 346/**347 * spum_nsp_ctx_max_payload() - Determine the max length of the payload for a348 * SPU message for a given cipher and hash alg context.349 * @cipher_alg: The cipher algorithm350 * @cipher_mode: The cipher mode351 * @blocksize: The size of a block of data for this algo352 *353 * The max payload must be a multiple of the blocksize so that if a request is354 * too large to fit in a single SPU message, the request can be broken into355 * max_payload sized chunks. Each chunk must be a multiple of blocksize.356 *357 * Return: Max payload length in bytes358 */359u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg,360 enum spu_cipher_mode cipher_mode,361 unsigned int blocksize)362{363 u32 max_payload = SPUM_NSP_MAX_PAYLOAD;364 u32 excess;365 366 /* In XTS on SPU-M, we'll need to insert tweak before input data */367 if (cipher_mode == CIPHER_MODE_XTS)368 max_payload -= SPU_XTS_TWEAK_SIZE;369 370 excess = max_payload % blocksize;371 372 return max_payload - excess;373}374 375/** spum_payload_length() - Given a SPU-M message header, extract the payload376 * length.377 * @spu_hdr: Start of SPU header378 *379 * Assumes just MH, EMH, BD (no SCTX, BDESC. Works for response frames.380 *381 * Return: payload length in bytes382 */383u32 spum_payload_length(u8 *spu_hdr)384{385 struct BD_HEADER *bd;386 u32 pl_len;387 388 /* Find BD header. skip MH, EMH */389 bd = (struct BD_HEADER *)(spu_hdr + 8);390 pl_len = be16_to_cpu(bd->size);391 392 return pl_len;393}394 395/**396 * spum_response_hdr_len() - Given the length of the hash key and encryption397 * key, determine the expected length of a SPU response header.398 * @auth_key_len: authentication key length (bytes)399 * @enc_key_len: encryption key length (bytes)400 * @is_hash: true if response message is for a hash operation401 *402 * Return: length of SPU response header (bytes)403 */404u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)405{406 if (is_hash)407 return SPU_HASH_RESP_HDR_LEN;408 else409 return SPU_RESP_HDR_LEN;410}411 412/**413 * spum_hash_pad_len() - Calculate the length of hash padding required to extend414 * data to a full block size.415 * @hash_alg: hash algorithm416 * @hash_mode: hash mode417 * @chunksize: length of data, in bytes418 * @hash_block_size: size of a block of data for hash algorithm419 *420 * Reserve space for 1 byte (0x80) start of pad and the total length as u64421 *422 * Return: length of hash pad in bytes423 */424u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,425 u32 chunksize, u16 hash_block_size)426{427 unsigned int length_len;428 unsigned int used_space_last_block;429 int hash_pad_len;430 431 /* AES-XCBC hash requires just padding to next block boundary */432 if ((hash_alg == HASH_ALG_AES) && (hash_mode == HASH_MODE_XCBC)) {433 used_space_last_block = chunksize % hash_block_size;434 hash_pad_len = hash_block_size - used_space_last_block;435 if (hash_pad_len >= hash_block_size)436 hash_pad_len -= hash_block_size;437 return hash_pad_len;438 }439 440 used_space_last_block = chunksize % hash_block_size + 1;441 if ((hash_alg == HASH_ALG_SHA384) || (hash_alg == HASH_ALG_SHA512))442 length_len = 2 * sizeof(u64);443 else444 length_len = sizeof(u64);445 446 used_space_last_block += length_len;447 hash_pad_len = hash_block_size - used_space_last_block;448 if (hash_pad_len < 0)449 hash_pad_len += hash_block_size;450 451 hash_pad_len += 1 + length_len;452 return hash_pad_len;453}454 455/**456 * spum_gcm_ccm_pad_len() - Determine the required length of GCM or CCM padding.457 * @cipher_mode: Algo type458 * @data_size: Length of plaintext (bytes)459 *460 * Return: Length of padding, in bytes461 */462u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,463 unsigned int data_size)464{465 u32 pad_len = 0;466 u32 m1 = SPU_GCM_CCM_ALIGN - 1;467 468 if ((cipher_mode == CIPHER_MODE_GCM) ||469 (cipher_mode == CIPHER_MODE_CCM))470 pad_len = ((data_size + m1) & ~m1) - data_size;471 472 return pad_len;473}474 475/**476 * spum_assoc_resp_len() - Determine the size of the receive buffer required to477 * catch associated data.478 * @cipher_mode: cipher mode479 * @assoc_len: length of associated data (bytes)480 * @iv_len: length of IV (bytes)481 * @is_encrypt: true if encrypting. false if decrypting.482 *483 * Return: length of associated data in response message (bytes)484 */485u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode,486 unsigned int assoc_len, unsigned int iv_len,487 bool is_encrypt)488{489 u32 buflen = 0;490 u32 pad;491 492 if (assoc_len)493 buflen = assoc_len;494 495 if (cipher_mode == CIPHER_MODE_GCM) {496 /* AAD needs to be padded in responses too */497 pad = spum_gcm_ccm_pad_len(cipher_mode, buflen);498 buflen += pad;499 }500 if (cipher_mode == CIPHER_MODE_CCM) {501 /*502 * AAD needs to be padded in responses too503 * for CCM, len + 2 needs to be 128-bit aligned.504 */505 pad = spum_gcm_ccm_pad_len(cipher_mode, buflen + 2);506 buflen += pad;507 }508 509 return buflen;510}511 512/**513 * spum_aead_ivlen() - Calculate the length of the AEAD IV to be included514 * in a SPU request after the AAD and before the payload.515 * @cipher_mode: cipher mode516 * @iv_len: initialization vector length in bytes517 *518 * In Linux ~4.2 and later, the assoc_data sg includes the IV. So no need519 * to include the IV as a separate field in the SPU request msg.520 *521 * Return: Length of AEAD IV in bytes522 */523u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)524{525 return 0;526}527 528/**529 * spum_hash_type() - Determine the type of hash operation.530 * @src_sent: The number of bytes in the current request that have already531 * been sent to the SPU to be hashed.532 *533 * We do not use HASH_TYPE_FULL for requests that fit in a single SPU message.534 * Using FULL causes failures (such as when the string to be hashed is empty).535 * For similar reasons, we never use HASH_TYPE_FIN. Instead, submit messages536 * as INIT or UPDT and do the hash padding in sw.537 */538enum hash_type spum_hash_type(u32 src_sent)539{540 return src_sent ? HASH_TYPE_UPDT : HASH_TYPE_INIT;541}542 543/**544 * spum_digest_size() - Determine the size of a hash digest to expect the SPU to545 * return.546 * @alg_digest_size: Number of bytes in the final digest for the given algo547 * @alg: The hash algorithm548 * @htype: Type of hash operation (init, update, full, etc)549 *550 * When doing incremental hashing for an algorithm with a truncated hash551 * (e.g., SHA224), the SPU returns the full digest so that it can be fed back as552 * a partial result for the next chunk.553 */554u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg,555 enum hash_type htype)556{557 u32 digestsize = alg_digest_size;558 559 /* SPU returns complete digest when doing incremental hash and truncated560 * hash algo.561 */562 if ((htype == HASH_TYPE_INIT) || (htype == HASH_TYPE_UPDT)) {563 if (alg == HASH_ALG_SHA224)564 digestsize = SHA256_DIGEST_SIZE;565 else if (alg == HASH_ALG_SHA384)566 digestsize = SHA512_DIGEST_SIZE;567 }568 return digestsize;569}570 571/**572 * spum_create_request() - Build a SPU request message header, up to and573 * including the BD header. Construct the message starting at spu_hdr. Caller574 * should allocate this buffer in DMA-able memory at least SPU_HEADER_ALLOC_LEN575 * bytes long.576 * @spu_hdr: Start of buffer where SPU request header is to be written577 * @req_opts: SPU request message options578 * @cipher_parms: Parameters related to cipher algorithm579 * @hash_parms: Parameters related to hash algorithm580 * @aead_parms: Parameters related to AEAD operation581 * @data_size: Length of data to be encrypted or authenticated. If AEAD, does582 * not include length of AAD.583 *584 * Return: the length of the SPU header in bytes. 0 if an error occurs.585 */586u32 spum_create_request(u8 *spu_hdr,587 struct spu_request_opts *req_opts,588 struct spu_cipher_parms *cipher_parms,589 struct spu_hash_parms *hash_parms,590 struct spu_aead_parms *aead_parms,591 unsigned int data_size)592{593 struct SPUHEADER *spuh;594 struct BDESC_HEADER *bdesc;595 struct BD_HEADER *bd;596 597 u8 *ptr;598 u32 protocol_bits = 0;599 u32 cipher_bits = 0;600 u32 ecf_bits = 0;601 u8 sctx_words = 0;602 unsigned int buf_len = 0;603 604 /* size of the cipher payload */605 unsigned int cipher_len = hash_parms->prebuf_len + data_size +606 hash_parms->pad_len;607 608 /* offset of prebuf or data from end of BD header */609 unsigned int cipher_offset = aead_parms->assoc_size +610 aead_parms->iv_len + aead_parms->aad_pad_len;611 612 /* total size of the DB data (without STAT word padding) */613 unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,614 aead_parms->iv_len,615 hash_parms->prebuf_len,616 data_size,617 aead_parms->aad_pad_len,618 aead_parms->data_pad_len,619 hash_parms->pad_len);620 621 unsigned int auth_offset = 0;622 unsigned int offset_iv = 0;623 624 /* size/offset of the auth payload */625 unsigned int auth_len;626 627 auth_len = real_db_size;628 629 if (req_opts->is_aead && req_opts->is_inbound)630 cipher_len -= hash_parms->digestsize;631 632 if (req_opts->is_aead && req_opts->is_inbound)633 auth_len -= hash_parms->digestsize;634 635 if ((hash_parms->alg == HASH_ALG_AES) &&636 (hash_parms->mode == HASH_MODE_XCBC)) {637 auth_len -= hash_parms->pad_len;638 cipher_len -= hash_parms->pad_len;639 }640 641 flow_log("%s()\n", __func__);642 flow_log(" in:%u authFirst:%u\n",643 req_opts->is_inbound, req_opts->auth_first);644 flow_log(" %s. cipher alg:%u mode:%u type %u\n",645 spu_alg_name(cipher_parms->alg, cipher_parms->mode),646 cipher_parms->alg, cipher_parms->mode, cipher_parms->type);647 flow_log(" key: %d\n", cipher_parms->key_len);648 flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);649 flow_log(" iv: %d\n", cipher_parms->iv_len);650 flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);651 flow_log(" auth alg:%u mode:%u type %u\n",652 hash_parms->alg, hash_parms->mode, hash_parms->type);653 flow_log(" digestsize: %u\n", hash_parms->digestsize);654 flow_log(" authkey: %d\n", hash_parms->key_len);655 flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len);656 flow_log(" assoc_size:%u\n", aead_parms->assoc_size);657 flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len);658 flow_log(" data_size:%u\n", data_size);659 flow_log(" hash_pad_len:%u\n", hash_parms->pad_len);660 flow_log(" real_db_size:%u\n", real_db_size);661 flow_log(" auth_offset:%u auth_len:%u cipher_offset:%u cipher_len:%u\n",662 auth_offset, auth_len, cipher_offset, cipher_len);663 flow_log(" aead_iv: %u\n", aead_parms->iv_len);664 665 /* starting out: zero the header (plus some) */666 ptr = spu_hdr;667 memset(ptr, 0, sizeof(struct SPUHEADER));668 669 /* format master header word */670 /* Do not set the next bit even though the datasheet says to */671 spuh = (struct SPUHEADER *)ptr;672 ptr += sizeof(struct SPUHEADER);673 buf_len += sizeof(struct SPUHEADER);674 675 spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;676 spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);677 678 /* Format sctx word 0 (protocol_bits) */679 sctx_words = 3; /* size in words */680 681 /* Format sctx word 1 (cipher_bits) */682 if (req_opts->is_inbound)683 cipher_bits |= CIPHER_INBOUND;684 if (req_opts->auth_first)685 cipher_bits |= CIPHER_ORDER;686 687 /* Set the crypto parameters in the cipher.flags */688 cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;689 cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;690 cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;691 692 /* Set the auth parameters in the cipher.flags */693 cipher_bits |= hash_parms->alg << HASH_ALG_SHIFT;694 cipher_bits |= hash_parms->mode << HASH_MODE_SHIFT;695 cipher_bits |= hash_parms->type << HASH_TYPE_SHIFT;696 697 /*698 * Format sctx extensions if required, and update main fields if699 * required)700 */701 if (hash_parms->alg) {702 /* Write the authentication key material if present */703 if (hash_parms->key_len) {704 memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);705 ptr += hash_parms->key_len;706 buf_len += hash_parms->key_len;707 sctx_words += hash_parms->key_len / 4;708 }709 710 if ((cipher_parms->mode == CIPHER_MODE_GCM) ||711 (cipher_parms->mode == CIPHER_MODE_CCM))712 /* unpadded length */713 offset_iv = aead_parms->assoc_size;714 715 /* if GCM/CCM we need to write ICV into the payload */716 if (!req_opts->is_inbound) {717 if ((cipher_parms->mode == CIPHER_MODE_GCM) ||718 (cipher_parms->mode == CIPHER_MODE_CCM))719 ecf_bits |= 1 << INSERT_ICV_SHIFT;720 } else {721 ecf_bits |= CHECK_ICV;722 }723 724 /* Inform the SPU of the ICV size (in words) */725 if (hash_parms->digestsize == 64)726 cipher_bits |= ICV_IS_512;727 else728 ecf_bits |=729 (hash_parms->digestsize / 4) << ICV_SIZE_SHIFT;730 }731 732 if (req_opts->bd_suppress)733 ecf_bits |= BD_SUPPRESS;734 735 /* copy the encryption keys in the SAD entry */736 if (cipher_parms->alg) {737 if (cipher_parms->key_len) {738 memcpy(ptr, cipher_parms->key_buf,739 cipher_parms->key_len);740 ptr += cipher_parms->key_len;741 buf_len += cipher_parms->key_len;742 sctx_words += cipher_parms->key_len / 4;743 }744 745 /*746 * if encrypting then set IV size, use SCTX IV unless no IV747 * given here748 */749 if (cipher_parms->iv_buf && cipher_parms->iv_len) {750 /* Use SCTX IV */751 ecf_bits |= SCTX_IV;752 753 /* cipher iv provided so put it in here */754 memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);755 756 ptr += cipher_parms->iv_len;757 buf_len += cipher_parms->iv_len;758 sctx_words += cipher_parms->iv_len / 4;759 }760 }761 762 /*763 * RFC4543 (GMAC/ESP) requires data to be sent as part of AAD764 * so we need to override the BDESC parameters.765 */766 if (req_opts->is_rfc4543) {767 if (req_opts->is_inbound)768 data_size -= hash_parms->digestsize;769 offset_iv = aead_parms->assoc_size + data_size;770 cipher_len = 0;771 cipher_offset = offset_iv;772 auth_len = cipher_offset + aead_parms->data_pad_len;773 }774 775 /* write in the total sctx length now that we know it */776 protocol_bits |= sctx_words;777 778 /* Endian adjust the SCTX */779 spuh->sa.proto_flags = cpu_to_be32(protocol_bits);780 spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);781 spuh->sa.ecf = cpu_to_be32(ecf_bits);782 783 /* === create the BDESC section === */784 bdesc = (struct BDESC_HEADER *)ptr;785 786 bdesc->offset_mac = cpu_to_be16(auth_offset);787 bdesc->length_mac = cpu_to_be16(auth_len);788 bdesc->offset_crypto = cpu_to_be16(cipher_offset);789 bdesc->length_crypto = cpu_to_be16(cipher_len);790 791 /*792 * CCM in SPU-M requires that ICV not be in same 32-bit word as data or793 * padding. So account for padding as necessary.794 */795 if (cipher_parms->mode == CIPHER_MODE_CCM)796 auth_len += spum_wordalign_padlen(auth_len);797 798 bdesc->offset_icv = cpu_to_be16(auth_len);799 bdesc->offset_iv = cpu_to_be16(offset_iv);800 801 ptr += sizeof(struct BDESC_HEADER);802 buf_len += sizeof(struct BDESC_HEADER);803 804 /* === no MFM section === */805 806 /* === create the BD section === */807 808 /* add the BD header */809 bd = (struct BD_HEADER *)ptr;810 bd->size = cpu_to_be16(real_db_size);811 bd->prev_length = 0;812 813 ptr += sizeof(struct BD_HEADER);814 buf_len += sizeof(struct BD_HEADER);815 816 packet_dump(" SPU request header: ", spu_hdr, buf_len);817 818 return buf_len;819}820 821/**822 * spum_cipher_req_init() - Build a SPU request message header, up to and823 * including the BD header.824 * @spu_hdr: Start of SPU request header (MH)825 * @cipher_parms: Parameters that describe the cipher request826 *827 * Construct the message starting at spu_hdr. Caller should allocate this buffer828 * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.829 *830 * Return: the length of the SPU header in bytes. 0 if an error occurs.831 */832u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)833{834 struct SPUHEADER *spuh;835 u32 protocol_bits = 0;836 u32 cipher_bits = 0;837 u32 ecf_bits = 0;838 u8 sctx_words = 0;839 u8 *ptr = spu_hdr;840 841 flow_log("%s()\n", __func__);842 flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,843 cipher_parms->mode, cipher_parms->type);844 flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len);845 flow_log(" key: %d\n", cipher_parms->key_len);846 flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);847 848 /* starting out: zero the header (plus some) */849 memset(spu_hdr, 0, sizeof(struct SPUHEADER));850 ptr += sizeof(struct SPUHEADER);851 852 /* format master header word */853 /* Do not set the next bit even though the datasheet says to */854 spuh = (struct SPUHEADER *)spu_hdr;855 856 spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;857 spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);858 859 /* Format sctx word 0 (protocol_bits) */860 sctx_words = 3; /* size in words */861 862 /* copy the encryption keys in the SAD entry */863 if (cipher_parms->alg) {864 if (cipher_parms->key_len) {865 ptr += cipher_parms->key_len;866 sctx_words += cipher_parms->key_len / 4;867 }868 869 /*870 * if encrypting then set IV size, use SCTX IV unless no IV871 * given here872 */873 if (cipher_parms->iv_len) {874 /* Use SCTX IV */875 ecf_bits |= SCTX_IV;876 ptr += cipher_parms->iv_len;877 sctx_words += cipher_parms->iv_len / 4;878 }879 }880 881 /* Set the crypto parameters in the cipher.flags */882 cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;883 cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;884 cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;885 886 /* copy the encryption keys in the SAD entry */887 if (cipher_parms->alg && cipher_parms->key_len)888 memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);889 890 /* write in the total sctx length now that we know it */891 protocol_bits |= sctx_words;892 893 /* Endian adjust the SCTX */894 spuh->sa.proto_flags = cpu_to_be32(protocol_bits);895 896 /* Endian adjust the SCTX */897 spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);898 spuh->sa.ecf = cpu_to_be32(ecf_bits);899 900 packet_dump(" SPU request header: ", spu_hdr,901 sizeof(struct SPUHEADER));902 903 return sizeof(struct SPUHEADER) + cipher_parms->key_len +904 cipher_parms->iv_len + sizeof(struct BDESC_HEADER) +905 sizeof(struct BD_HEADER);906}907 908/**909 * spum_cipher_req_finish() - Finish building a SPU request message header for a910 * block cipher request. Assumes much of the header was already filled in at911 * setkey() time in spu_cipher_req_init().912 * @spu_hdr: Start of the request message header (MH field)913 * @spu_req_hdr_len: Length in bytes of the SPU request header914 * @is_inbound: 0 encrypt, 1 decrypt915 * @cipher_parms: Parameters describing cipher operation to be performed916 * @data_size: Length of the data in the BD field917 *918 * Assumes much of the header was already filled in at setkey() time in919 * spum_cipher_req_init().920 * spum_cipher_req_init() fills in the encryption key.921 */922void spum_cipher_req_finish(u8 *spu_hdr,923 u16 spu_req_hdr_len,924 unsigned int is_inbound,925 struct spu_cipher_parms *cipher_parms,926 unsigned int data_size)927{928 struct SPUHEADER *spuh;929 struct BDESC_HEADER *bdesc;930 struct BD_HEADER *bd;931 u8 *bdesc_ptr = spu_hdr + spu_req_hdr_len -932 (sizeof(struct BD_HEADER) + sizeof(struct BDESC_HEADER));933 934 u32 cipher_bits;935 936 flow_log("%s()\n", __func__);937 flow_log(" in: %u\n", is_inbound);938 flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,939 cipher_parms->type);940 941 /*942 * In XTS mode, API puts "i" parameter (block tweak) in IV. For943 * SPU-M, should be in start of the BD; tx_sg_create() copies it there.944 * IV in SPU msg for SPU-M should be 0, since that's the "j" parameter945 * (block ctr within larger data unit) - given we can send entire disk946 * block (<= 4KB) in 1 SPU msg, don't need to use this parameter.947 */948 if (cipher_parms->mode == CIPHER_MODE_XTS)949 memset(cipher_parms->iv_buf, 0, cipher_parms->iv_len);950 951 flow_log(" iv len: %d\n", cipher_parms->iv_len);952 flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);953 flow_log(" data_size: %u\n", data_size);954 955 /* format master header word */956 /* Do not set the next bit even though the datasheet says to */957 spuh = (struct SPUHEADER *)spu_hdr;958 959 /* cipher_bits was initialized at setkey time */960 cipher_bits = be32_to_cpu(spuh->sa.cipher_flags);961 962 /* Format sctx word 1 (cipher_bits) */963 if (is_inbound)964 cipher_bits |= CIPHER_INBOUND;965 else966 cipher_bits &= ~CIPHER_INBOUND;967 968 if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len)969 /* cipher iv provided so put it in here */970 memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf,971 cipher_parms->iv_len);972 973 spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);974 975 /* === create the BDESC section === */976 bdesc = (struct BDESC_HEADER *)bdesc_ptr;977 bdesc->offset_mac = 0;978 bdesc->length_mac = 0;979 bdesc->offset_crypto = 0;980 981 /* XTS mode, data_size needs to include tweak parameter */982 if (cipher_parms->mode == CIPHER_MODE_XTS)983 bdesc->length_crypto = cpu_to_be16(data_size +984 SPU_XTS_TWEAK_SIZE);985 else986 bdesc->length_crypto = cpu_to_be16(data_size);987 988 bdesc->offset_icv = 0;989 bdesc->offset_iv = 0;990 991 /* === no MFM section === */992 993 /* === create the BD section === */994 /* add the BD header */995 bd = (struct BD_HEADER *)(bdesc_ptr + sizeof(struct BDESC_HEADER));996 bd->size = cpu_to_be16(data_size);997 998 /* XTS mode, data_size needs to include tweak parameter */999 if (cipher_parms->mode == CIPHER_MODE_XTS)1000 bd->size = cpu_to_be16(data_size + SPU_XTS_TWEAK_SIZE);1001 else1002 bd->size = cpu_to_be16(data_size);1003 1004 bd->prev_length = 0;1005 1006 packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len);1007}1008 1009/**1010 * spum_request_pad() - Create pad bytes at the end of the data.1011 * @pad_start: Start of buffer where pad bytes are to be written1012 * @gcm_ccm_padding: length of GCM/CCM padding, in bytes1013 * @hash_pad_len: Number of bytes of padding extend data to full block1014 * @auth_alg: authentication algorithm1015 * @auth_mode: authentication mode1016 * @total_sent: length inserted at end of hash pad1017 * @status_padding: Number of bytes of padding to align STATUS word1018 *1019 * There may be three forms of pad:1020 * 1. GCM/CCM pad - for GCM/CCM mode ciphers, pad to 16-byte alignment1021 * 2. hash pad - pad to a block length, with 0x80 data terminator and1022 * size at the end1023 * 3. STAT pad - to ensure the STAT field is 4-byte aligned1024 */1025void spum_request_pad(u8 *pad_start,1026 u32 gcm_ccm_padding,1027 u32 hash_pad_len,1028 enum hash_alg auth_alg,1029 enum hash_mode auth_mode,1030 unsigned int total_sent, u32 status_padding)1031{1032 u8 *ptr = pad_start;1033 1034 /* fix data alignent for GCM/CCM */1035 if (gcm_ccm_padding > 0) {1036 flow_log(" GCM: padding to 16 byte alignment: %u bytes\n",1037 gcm_ccm_padding);1038 memset(ptr, 0, gcm_ccm_padding);1039 ptr += gcm_ccm_padding;1040 }1041 1042 if (hash_pad_len > 0) {1043 /* clear the padding section */1044 memset(ptr, 0, hash_pad_len);1045 1046 if ((auth_alg == HASH_ALG_AES) &&1047 (auth_mode == HASH_MODE_XCBC)) {1048 /* AES/XCBC just requires padding to be 0s */1049 ptr += hash_pad_len;1050 } else {1051 /* terminate the data */1052 *ptr = 0x80;1053 ptr += (hash_pad_len - sizeof(u64));1054 1055 /* add the size at the end as required per alg */1056 if (auth_alg == HASH_ALG_MD5)1057 *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull);1058 else /* SHA1, SHA2-224, SHA2-256 */1059 *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull);1060 ptr += sizeof(u64);1061 }1062 }1063 1064 /* pad to a 4byte alignment for STAT */1065 if (status_padding > 0) {1066 flow_log(" STAT: padding to 4 byte alignment: %u bytes\n",1067 status_padding);1068 1069 memset(ptr, 0, status_padding);1070 ptr += status_padding;1071 }1072}1073 1074/**1075 * spum_xts_tweak_in_payload() - Indicate that SPUM DOES place the XTS tweak1076 * field in the packet payload (rather than using IV)1077 *1078 * Return: 11079 */1080u8 spum_xts_tweak_in_payload(void)1081{1082 return 1;1083}1084 1085/**1086 * spum_tx_status_len() - Return the length of the STATUS field in a SPU1087 * response message.1088 *1089 * Return: Length of STATUS field in bytes.1090 */1091u8 spum_tx_status_len(void)1092{1093 return SPU_TX_STATUS_LEN;1094}1095 1096/**1097 * spum_rx_status_len() - Return the length of the STATUS field in a SPU1098 * response message.1099 *1100 * Return: Length of STATUS field in bytes.1101 */1102u8 spum_rx_status_len(void)1103{1104 return SPU_RX_STATUS_LEN;1105}1106 1107/**1108 * spum_status_process() - Process the status from a SPU response message.1109 * @statp: start of STATUS word1110 * Return:1111 * 0 - if status is good and response should be processed1112 * !0 - status indicates an error and response is invalid1113 */1114int spum_status_process(u8 *statp)1115{1116 u32 status;1117 1118 status = __be32_to_cpu(*(__be32 *)statp);1119 flow_log("SPU response STATUS %#08x\n", status);1120 if (status & SPU_STATUS_ERROR_FLAG) {1121 pr_err("%s() Warning: Error result from SPU: %#08x\n",1122 __func__, status);1123 if (status & SPU_STATUS_INVALID_ICV)1124 return SPU_INVALID_ICV;1125 return -EBADMSG;1126 }1127 return 0;1128}1129 1130/**1131 * spum_ccm_update_iv() - Update the IV as per the requirements for CCM mode.1132 *1133 * @digestsize: Digest size of this request1134 * @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len1135 * @assoclen: Length of AAD data1136 * @chunksize: length of input data to be sent in this req1137 * @is_encrypt: true if this is an output/encrypt operation1138 * @is_esp: true if this is an ESP / RFC4309 operation1139 *1140 */1141void spum_ccm_update_iv(unsigned int digestsize,1142 struct spu_cipher_parms *cipher_parms,1143 unsigned int assoclen,1144 unsigned int chunksize,1145 bool is_encrypt,1146 bool is_esp)1147{1148 u8 L; /* L from CCM algorithm, length of plaintext data */1149 u8 mprime; /* M' from CCM algo, (M - 2) / 2, where M=authsize */1150 u8 adata;1151 1152 if (cipher_parms->iv_len != CCM_AES_IV_SIZE) {1153 pr_err("%s(): Invalid IV len %d for CCM mode, should be %d\n",1154 __func__, cipher_parms->iv_len, CCM_AES_IV_SIZE);1155 return;1156 }1157 1158 /*1159 * IV needs to be formatted as follows:1160 *1161 * | Byte 0 | Bytes 1 - N | Bytes (N+1) - 15 |1162 * | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | Bits 7 - 0 | Bits 7 - 0 |1163 * | 0 |Ad?|(M - 2) / 2| L - 1 | Nonce | Plaintext Length |1164 *1165 * Ad? = 1 if AAD present, 0 if not present1166 * M = size of auth field, 8, 12, or 16 bytes (SPU-M) -or-1167 * 4, 6, 8, 10, 12, 14, 16 bytes (SPU2)1168 * L = Size of Plaintext Length field; Nonce size = 15 - L1169 *1170 * It appears that the crypto API already expects the L-1 portion1171 * to be set in the first byte of the IV, which implicitly determines1172 * the nonce size, and also fills in the nonce. But the other bits1173 * in byte 0 as well as the plaintext length need to be filled in.1174 *1175 * In rfc4309/esp mode, L is not already in the supplied IV and1176 * we need to fill it in, as well as move the IV data to be after1177 * the salt1178 */1179 if (is_esp) {1180 L = CCM_ESP_L_VALUE; /* RFC4309 has fixed L */1181 } else {1182 /* L' = plaintext length - 1 so Plaintext length is L' + 1 */1183 L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>1184 CCM_B0_L_PRIME_SHIFT) + 1;1185 }1186 1187 mprime = (digestsize - 2) >> 1; /* M' = (M - 2) / 2 */1188 adata = (assoclen > 0); /* adata = 1 if any associated data */1189 1190 cipher_parms->iv_buf[0] = (adata << CCM_B0_ADATA_SHIFT) |1191 (mprime << CCM_B0_M_PRIME_SHIFT) |1192 ((L - 1) << CCM_B0_L_PRIME_SHIFT);1193 1194 /* Nonce is already filled in by crypto API, and is 15 - L bytes */1195 1196 /* Don't include digest in plaintext size when decrypting */1197 if (!is_encrypt)1198 chunksize -= digestsize;1199 1200 /* Fill in length of plaintext, formatted to be L bytes long */1201 format_value_ccm(chunksize, &cipher_parms->iv_buf[15 - L + 1], L);1202}1203 1204/**1205 * spum_wordalign_padlen() - Given the length of a data field, determine the1206 * padding required to align the data following this field on a 4-byte boundary.1207 * @data_size: length of data field in bytes1208 *1209 * Return: length of status field padding, in bytes1210 */1211u32 spum_wordalign_padlen(u32 data_size)1212{1213 return ((data_size + 3) & ~3) - data_size;1214}1215