2316 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */3 4#include <linux/kernel.h>5#include <linux/module.h>6#include <crypto/algapi.h>7#include <crypto/hash.h>8#include <crypto/md5.h>9#include <crypto/sm3.h>10#include <crypto/internal/hash.h>11 12#include "cc_driver.h"13#include "cc_request_mgr.h"14#include "cc_buffer_mgr.h"15#include "cc_hash.h"16#include "cc_sram_mgr.h"17 18#define CC_MAX_HASH_SEQ_LEN 1219#define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE20#define CC_SM3_HASH_LEN_SIZE 821 22struct cc_hash_handle {23 u32 digest_len_sram_addr; /* const value in SRAM*/24 u32 larval_digest_sram_addr; /* const value in SRAM */25 struct list_head hash_list;26};27 28static const u32 cc_digest_len_init[] = {29 0x00000040, 0x00000000, 0x00000000, 0x00000000 };30static const u32 cc_md5_init[] = {31 SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };32static const u32 cc_sha1_init[] = {33 SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };34static const u32 cc_sha224_init[] = {35 SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4,36 SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 };37static const u32 cc_sha256_init[] = {38 SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,39 SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };40static const u32 cc_digest_len_sha512_init[] = {41 0x00000080, 0x00000000, 0x00000000, 0x00000000 };42 43/*44 * Due to the way the HW works, every double word in the SHA384 and SHA51245 * larval hashes must be stored in hi/lo order46 */47#define hilo(x) upper_32_bits(x), lower_32_bits(x)48static const u32 cc_sha384_init[] = {49 hilo(SHA384_H7), hilo(SHA384_H6), hilo(SHA384_H5), hilo(SHA384_H4),50 hilo(SHA384_H3), hilo(SHA384_H2), hilo(SHA384_H1), hilo(SHA384_H0) };51static const u32 cc_sha512_init[] = {52 hilo(SHA512_H7), hilo(SHA512_H6), hilo(SHA512_H5), hilo(SHA512_H4),53 hilo(SHA512_H3), hilo(SHA512_H2), hilo(SHA512_H1), hilo(SHA512_H0) };54 55static const u32 cc_sm3_init[] = {56 SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE,57 SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA };58 59static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],60 unsigned int *seq_size);61 62static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],63 unsigned int *seq_size);64 65static const void *cc_larval_digest(struct device *dev, u32 mode);66 67struct cc_hash_alg {68 struct list_head entry;69 int hash_mode;70 int hw_mode;71 int inter_digestsize;72 struct cc_drvdata *drvdata;73 struct ahash_alg ahash_alg;74};75 76struct hash_key_req_ctx {77 u32 keylen;78 dma_addr_t key_dma_addr;79 u8 *key;80};81 82/* hash per-session context */83struct cc_hash_ctx {84 struct cc_drvdata *drvdata;85 /* holds the origin digest; the digest after "setkey" if HMAC,*86 * the initial digest if HASH.87 */88 u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned;89 u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE] ____cacheline_aligned;90 91 dma_addr_t opad_tmp_keys_dma_addr ____cacheline_aligned;92 dma_addr_t digest_buff_dma_addr;93 /* use for hmac with key large then mode block size */94 struct hash_key_req_ctx key_params;95 int hash_mode;96 int hw_mode;97 int inter_digestsize;98 unsigned int hash_len;99 struct completion setkey_comp;100 bool is_hmac;101};102 103static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx,104 unsigned int flow_mode, struct cc_hw_desc desc[],105 bool is_not_last_data, unsigned int *seq_size);106 107static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc)108{109 if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 ||110 mode == DRV_HASH_SHA512) {111 set_bytes_swap(desc, 1);112 } else {113 set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN);114 }115}116 117static int cc_map_result(struct device *dev, struct ahash_req_ctx *state,118 unsigned int digestsize)119{120 state->digest_result_dma_addr =121 dma_map_single(dev, state->digest_result_buff,122 digestsize, DMA_BIDIRECTIONAL);123 if (dma_mapping_error(dev, state->digest_result_dma_addr)) {124 dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n",125 digestsize);126 return -ENOMEM;127 }128 dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n",129 digestsize, state->digest_result_buff,130 &state->digest_result_dma_addr);131 132 return 0;133}134 135static void cc_init_req(struct device *dev, struct ahash_req_ctx *state,136 struct cc_hash_ctx *ctx)137{138 bool is_hmac = ctx->is_hmac;139 140 memset(state, 0, sizeof(*state));141 142 if (is_hmac) {143 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC &&144 ctx->hw_mode != DRV_CIPHER_CMAC) {145 dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr,146 ctx->inter_digestsize,147 DMA_BIDIRECTIONAL);148 149 memcpy(state->digest_buff, ctx->digest_buff,150 ctx->inter_digestsize);151 if (ctx->hash_mode == DRV_HASH_SHA512 ||152 ctx->hash_mode == DRV_HASH_SHA384)153 memcpy(state->digest_bytes_len,154 cc_digest_len_sha512_init,155 ctx->hash_len);156 else157 memcpy(state->digest_bytes_len,158 cc_digest_len_init,159 ctx->hash_len);160 }161 162 if (ctx->hash_mode != DRV_HASH_NULL) {163 dma_sync_single_for_cpu(dev,164 ctx->opad_tmp_keys_dma_addr,165 ctx->inter_digestsize,166 DMA_BIDIRECTIONAL);167 memcpy(state->opad_digest_buff,168 ctx->opad_tmp_keys_buff, ctx->inter_digestsize);169 }170 } else { /*hash*/171 /* Copy the initial digests if hash flow. */172 const void *larval = cc_larval_digest(dev, ctx->hash_mode);173 174 memcpy(state->digest_buff, larval, ctx->inter_digestsize);175 }176}177 178static int cc_map_req(struct device *dev, struct ahash_req_ctx *state,179 struct cc_hash_ctx *ctx)180{181 bool is_hmac = ctx->is_hmac;182 183 state->digest_buff_dma_addr =184 dma_map_single(dev, state->digest_buff,185 ctx->inter_digestsize, DMA_BIDIRECTIONAL);186 if (dma_mapping_error(dev, state->digest_buff_dma_addr)) {187 dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n",188 ctx->inter_digestsize, state->digest_buff);189 return -EINVAL;190 }191 dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n",192 ctx->inter_digestsize, state->digest_buff,193 &state->digest_buff_dma_addr);194 195 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) {196 state->digest_bytes_len_dma_addr =197 dma_map_single(dev, state->digest_bytes_len,198 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);199 if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) {200 dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n",201 HASH_MAX_LEN_SIZE, state->digest_bytes_len);202 goto unmap_digest_buf;203 }204 dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n",205 HASH_MAX_LEN_SIZE, state->digest_bytes_len,206 &state->digest_bytes_len_dma_addr);207 }208 209 if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) {210 state->opad_digest_dma_addr =211 dma_map_single(dev, state->opad_digest_buff,212 ctx->inter_digestsize,213 DMA_BIDIRECTIONAL);214 if (dma_mapping_error(dev, state->opad_digest_dma_addr)) {215 dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n",216 ctx->inter_digestsize,217 state->opad_digest_buff);218 goto unmap_digest_len;219 }220 dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n",221 ctx->inter_digestsize, state->opad_digest_buff,222 &state->opad_digest_dma_addr);223 }224 225 return 0;226 227unmap_digest_len:228 if (state->digest_bytes_len_dma_addr) {229 dma_unmap_single(dev, state->digest_bytes_len_dma_addr,230 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);231 state->digest_bytes_len_dma_addr = 0;232 }233unmap_digest_buf:234 if (state->digest_buff_dma_addr) {235 dma_unmap_single(dev, state->digest_buff_dma_addr,236 ctx->inter_digestsize, DMA_BIDIRECTIONAL);237 state->digest_buff_dma_addr = 0;238 }239 240 return -EINVAL;241}242 243static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state,244 struct cc_hash_ctx *ctx)245{246 if (state->digest_buff_dma_addr) {247 dma_unmap_single(dev, state->digest_buff_dma_addr,248 ctx->inter_digestsize, DMA_BIDIRECTIONAL);249 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",250 &state->digest_buff_dma_addr);251 state->digest_buff_dma_addr = 0;252 }253 if (state->digest_bytes_len_dma_addr) {254 dma_unmap_single(dev, state->digest_bytes_len_dma_addr,255 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);256 dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n",257 &state->digest_bytes_len_dma_addr);258 state->digest_bytes_len_dma_addr = 0;259 }260 if (state->opad_digest_dma_addr) {261 dma_unmap_single(dev, state->opad_digest_dma_addr,262 ctx->inter_digestsize, DMA_BIDIRECTIONAL);263 dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n",264 &state->opad_digest_dma_addr);265 state->opad_digest_dma_addr = 0;266 }267}268 269static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state,270 unsigned int digestsize, u8 *result)271{272 if (state->digest_result_dma_addr) {273 dma_unmap_single(dev, state->digest_result_dma_addr, digestsize,274 DMA_BIDIRECTIONAL);275 dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n",276 state->digest_result_buff,277 &state->digest_result_dma_addr, digestsize);278 memcpy(result, state->digest_result_buff, digestsize);279 }280 state->digest_result_dma_addr = 0;281}282 283static void cc_update_complete(struct device *dev, void *cc_req, int err)284{285 struct ahash_request *req = (struct ahash_request *)cc_req;286 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);287 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);288 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);289 290 dev_dbg(dev, "req=%pK\n", req);291 292 if (err != -EINPROGRESS) {293 /* Not a BACKLOG notification */294 cc_unmap_hash_request(dev, state, req->src, false);295 cc_unmap_req(dev, state, ctx);296 }297 298 ahash_request_complete(req, err);299}300 301static void cc_digest_complete(struct device *dev, void *cc_req, int err)302{303 struct ahash_request *req = (struct ahash_request *)cc_req;304 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);305 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);306 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);307 u32 digestsize = crypto_ahash_digestsize(tfm);308 309 dev_dbg(dev, "req=%pK\n", req);310 311 if (err != -EINPROGRESS) {312 /* Not a BACKLOG notification */313 cc_unmap_hash_request(dev, state, req->src, false);314 cc_unmap_result(dev, state, digestsize, req->result);315 cc_unmap_req(dev, state, ctx);316 }317 318 ahash_request_complete(req, err);319}320 321static void cc_hash_complete(struct device *dev, void *cc_req, int err)322{323 struct ahash_request *req = (struct ahash_request *)cc_req;324 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);325 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);326 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);327 u32 digestsize = crypto_ahash_digestsize(tfm);328 329 dev_dbg(dev, "req=%pK\n", req);330 331 if (err != -EINPROGRESS) {332 /* Not a BACKLOG notification */333 cc_unmap_hash_request(dev, state, req->src, false);334 cc_unmap_result(dev, state, digestsize, req->result);335 cc_unmap_req(dev, state, ctx);336 }337 338 ahash_request_complete(req, err);339}340 341static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req,342 int idx)343{344 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);346 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);347 u32 digestsize = crypto_ahash_digestsize(tfm);348 349 /* Get final MAC result */350 hw_desc_init(&desc[idx]);351 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);352 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize,353 NS_BIT, 1);354 set_queue_last_ind(ctx->drvdata, &desc[idx]);355 set_flow_mode(&desc[idx], S_HASH_to_DOUT);356 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);357 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);358 cc_set_endianity(ctx->hash_mode, &desc[idx]);359 idx++;360 361 return idx;362}363 364static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req,365 int idx)366{367 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);368 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);369 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);370 u32 digestsize = crypto_ahash_digestsize(tfm);371 372 /* store the hash digest result in the context */373 hw_desc_init(&desc[idx]);374 set_cipher_mode(&desc[idx], ctx->hw_mode);375 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize,376 NS_BIT, 0);377 set_flow_mode(&desc[idx], S_HASH_to_DOUT);378 cc_set_endianity(ctx->hash_mode, &desc[idx]);379 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);380 idx++;381 382 /* Loading hash opad xor key state */383 hw_desc_init(&desc[idx]);384 set_cipher_mode(&desc[idx], ctx->hw_mode);385 set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr,386 ctx->inter_digestsize, NS_BIT);387 set_flow_mode(&desc[idx], S_DIN_to_HASH);388 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);389 idx++;390 391 /* Load the hash current length */392 hw_desc_init(&desc[idx]);393 set_cipher_mode(&desc[idx], ctx->hw_mode);394 set_din_sram(&desc[idx],395 cc_digest_len_addr(ctx->drvdata, ctx->hash_mode),396 ctx->hash_len);397 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);398 set_flow_mode(&desc[idx], S_DIN_to_HASH);399 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);400 idx++;401 402 /* Memory Barrier: wait for IPAD/OPAD axi write to complete */403 hw_desc_init(&desc[idx]);404 set_din_no_dma(&desc[idx], 0, 0xfffff0);405 set_dout_no_dma(&desc[idx], 0, 0, 1);406 idx++;407 408 /* Perform HASH update */409 hw_desc_init(&desc[idx]);410 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,411 digestsize, NS_BIT);412 set_flow_mode(&desc[idx], DIN_HASH);413 idx++;414 415 return idx;416}417 418static int cc_hash_digest(struct ahash_request *req)419{420 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);421 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);422 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);423 u32 digestsize = crypto_ahash_digestsize(tfm);424 struct scatterlist *src = req->src;425 unsigned int nbytes = req->nbytes;426 u8 *result = req->result;427 struct device *dev = drvdata_to_dev(ctx->drvdata);428 bool is_hmac = ctx->is_hmac;429 struct cc_crypto_req cc_req = {};430 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];431 u32 larval_digest_addr;432 int idx = 0;433 int rc = 0;434 gfp_t flags = cc_gfp_flags(&req->base);435 436 dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash",437 nbytes);438 439 cc_init_req(dev, state, ctx);440 441 if (cc_map_req(dev, state, ctx)) {442 dev_err(dev, "map_ahash_source() failed\n");443 return -ENOMEM;444 }445 446 if (cc_map_result(dev, state, digestsize)) {447 dev_err(dev, "map_ahash_digest() failed\n");448 cc_unmap_req(dev, state, ctx);449 return -ENOMEM;450 }451 452 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1,453 flags)) {454 dev_err(dev, "map_ahash_request_final() failed\n");455 cc_unmap_result(dev, state, digestsize, result);456 cc_unmap_req(dev, state, ctx);457 return -ENOMEM;458 }459 460 /* Setup request structure */461 cc_req.user_cb = cc_digest_complete;462 cc_req.user_arg = req;463 464 /* If HMAC then load hash IPAD xor key, if HASH then load initial465 * digest466 */467 hw_desc_init(&desc[idx]);468 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);469 if (is_hmac) {470 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,471 ctx->inter_digestsize, NS_BIT);472 } else {473 larval_digest_addr = cc_larval_digest_addr(ctx->drvdata,474 ctx->hash_mode);475 set_din_sram(&desc[idx], larval_digest_addr,476 ctx->inter_digestsize);477 }478 set_flow_mode(&desc[idx], S_DIN_to_HASH);479 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);480 idx++;481 482 /* Load the hash current length */483 hw_desc_init(&desc[idx]);484 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);485 486 if (is_hmac) {487 set_din_type(&desc[idx], DMA_DLLI,488 state->digest_bytes_len_dma_addr,489 ctx->hash_len, NS_BIT);490 } else {491 set_din_const(&desc[idx], 0, ctx->hash_len);492 if (nbytes)493 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);494 else495 set_cipher_do(&desc[idx], DO_PAD);496 }497 set_flow_mode(&desc[idx], S_DIN_to_HASH);498 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);499 idx++;500 501 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);502 503 if (is_hmac) {504 /* HW last hash block padding (aka. "DO_PAD") */505 hw_desc_init(&desc[idx]);506 set_cipher_mode(&desc[idx], ctx->hw_mode);507 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,508 ctx->hash_len, NS_BIT, 0);509 set_flow_mode(&desc[idx], S_HASH_to_DOUT);510 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);511 set_cipher_do(&desc[idx], DO_PAD);512 idx++;513 514 idx = cc_fin_hmac(desc, req, idx);515 }516 517 idx = cc_fin_result(desc, req, idx);518 519 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);520 if (rc != -EINPROGRESS && rc != -EBUSY) {521 dev_err(dev, "send_request() failed (rc=%d)\n", rc);522 cc_unmap_hash_request(dev, state, src, true);523 cc_unmap_result(dev, state, digestsize, result);524 cc_unmap_req(dev, state, ctx);525 }526 return rc;527}528 529static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx,530 struct ahash_req_ctx *state, unsigned int idx)531{532 /* Restore hash digest */533 hw_desc_init(&desc[idx]);534 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);535 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,536 ctx->inter_digestsize, NS_BIT);537 set_flow_mode(&desc[idx], S_DIN_to_HASH);538 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);539 idx++;540 541 /* Restore hash current length */542 hw_desc_init(&desc[idx]);543 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);544 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);545 set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr,546 ctx->hash_len, NS_BIT);547 set_flow_mode(&desc[idx], S_DIN_to_HASH);548 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);549 idx++;550 551 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);552 553 return idx;554}555 556static int cc_hash_update(struct ahash_request *req)557{558 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);559 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);560 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);561 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);562 struct scatterlist *src = req->src;563 unsigned int nbytes = req->nbytes;564 struct device *dev = drvdata_to_dev(ctx->drvdata);565 struct cc_crypto_req cc_req = {};566 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];567 u32 idx = 0;568 int rc;569 gfp_t flags = cc_gfp_flags(&req->base);570 571 dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ?572 "hmac" : "hash", nbytes);573 574 if (nbytes == 0) {575 /* no real updates required */576 return 0;577 }578 579 rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes,580 block_size, flags);581 if (rc) {582 if (rc == 1) {583 dev_dbg(dev, " data size not require HW update %x\n",584 nbytes);585 /* No hardware updates are required */586 return 0;587 }588 dev_err(dev, "map_ahash_request_update() failed\n");589 return -ENOMEM;590 }591 592 if (cc_map_req(dev, state, ctx)) {593 dev_err(dev, "map_ahash_source() failed\n");594 cc_unmap_hash_request(dev, state, src, true);595 return -EINVAL;596 }597 598 /* Setup request structure */599 cc_req.user_cb = cc_update_complete;600 cc_req.user_arg = req;601 602 idx = cc_restore_hash(desc, ctx, state, idx);603 604 /* store the hash digest result in context */605 hw_desc_init(&desc[idx]);606 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);607 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,608 ctx->inter_digestsize, NS_BIT, 0);609 set_flow_mode(&desc[idx], S_HASH_to_DOUT);610 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);611 idx++;612 613 /* store current hash length in context */614 hw_desc_init(&desc[idx]);615 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);616 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,617 ctx->hash_len, NS_BIT, 1);618 set_queue_last_ind(ctx->drvdata, &desc[idx]);619 set_flow_mode(&desc[idx], S_HASH_to_DOUT);620 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);621 idx++;622 623 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);624 if (rc != -EINPROGRESS && rc != -EBUSY) {625 dev_err(dev, "send_request() failed (rc=%d)\n", rc);626 cc_unmap_hash_request(dev, state, src, true);627 cc_unmap_req(dev, state, ctx);628 }629 return rc;630}631 632static int cc_do_finup(struct ahash_request *req, bool update)633{634 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);635 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);636 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);637 u32 digestsize = crypto_ahash_digestsize(tfm);638 struct scatterlist *src = req->src;639 unsigned int nbytes = req->nbytes;640 u8 *result = req->result;641 struct device *dev = drvdata_to_dev(ctx->drvdata);642 bool is_hmac = ctx->is_hmac;643 struct cc_crypto_req cc_req = {};644 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];645 unsigned int idx = 0;646 int rc;647 gfp_t flags = cc_gfp_flags(&req->base);648 649 dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash",650 update ? "finup" : "final", nbytes);651 652 if (cc_map_req(dev, state, ctx)) {653 dev_err(dev, "map_ahash_source() failed\n");654 return -EINVAL;655 }656 657 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update,658 flags)) {659 dev_err(dev, "map_ahash_request_final() failed\n");660 cc_unmap_req(dev, state, ctx);661 return -ENOMEM;662 }663 if (cc_map_result(dev, state, digestsize)) {664 dev_err(dev, "map_ahash_digest() failed\n");665 cc_unmap_hash_request(dev, state, src, true);666 cc_unmap_req(dev, state, ctx);667 return -ENOMEM;668 }669 670 /* Setup request structure */671 cc_req.user_cb = cc_hash_complete;672 cc_req.user_arg = req;673 674 idx = cc_restore_hash(desc, ctx, state, idx);675 676 /* Pad the hash */677 hw_desc_init(&desc[idx]);678 set_cipher_do(&desc[idx], DO_PAD);679 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);680 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,681 ctx->hash_len, NS_BIT, 0);682 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);683 set_flow_mode(&desc[idx], S_HASH_to_DOUT);684 idx++;685 686 if (is_hmac)687 idx = cc_fin_hmac(desc, req, idx);688 689 idx = cc_fin_result(desc, req, idx);690 691 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);692 if (rc != -EINPROGRESS && rc != -EBUSY) {693 dev_err(dev, "send_request() failed (rc=%d)\n", rc);694 cc_unmap_hash_request(dev, state, src, true);695 cc_unmap_result(dev, state, digestsize, result);696 cc_unmap_req(dev, state, ctx);697 }698 return rc;699}700 701static int cc_hash_finup(struct ahash_request *req)702{703 return cc_do_finup(req, true);704}705 706 707static int cc_hash_final(struct ahash_request *req)708{709 return cc_do_finup(req, false);710}711 712static int cc_hash_init(struct ahash_request *req)713{714 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);715 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);716 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);717 struct device *dev = drvdata_to_dev(ctx->drvdata);718 719 dev_dbg(dev, "===== init (%d) ====\n", req->nbytes);720 721 cc_init_req(dev, state, ctx);722 723 return 0;724}725 726static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key,727 unsigned int keylen)728{729 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };730 struct cc_crypto_req cc_req = {};731 struct cc_hash_ctx *ctx = NULL;732 int blocksize = 0;733 int digestsize = 0;734 int i, idx = 0, rc = 0;735 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];736 u32 larval_addr;737 struct device *dev;738 739 ctx = crypto_ahash_ctx_dma(ahash);740 dev = drvdata_to_dev(ctx->drvdata);741 dev_dbg(dev, "start keylen: %d", keylen);742 743 blocksize = crypto_tfm_alg_blocksize(&ahash->base);744 digestsize = crypto_ahash_digestsize(ahash);745 746 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode);747 748 /* The keylen value distinguishes HASH in case keylen is ZERO bytes,749 * any NON-ZERO value utilizes HMAC flow750 */751 ctx->key_params.keylen = keylen;752 ctx->key_params.key_dma_addr = 0;753 ctx->is_hmac = true;754 ctx->key_params.key = NULL;755 756 if (keylen) {757 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL);758 if (!ctx->key_params.key)759 return -ENOMEM;760 761 ctx->key_params.key_dma_addr =762 dma_map_single(dev, ctx->key_params.key, keylen,763 DMA_TO_DEVICE);764 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {765 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",766 ctx->key_params.key, keylen);767 kfree_sensitive(ctx->key_params.key);768 return -ENOMEM;769 }770 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",771 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);772 773 if (keylen > blocksize) {774 /* Load hash initial state */775 hw_desc_init(&desc[idx]);776 set_cipher_mode(&desc[idx], ctx->hw_mode);777 set_din_sram(&desc[idx], larval_addr,778 ctx->inter_digestsize);779 set_flow_mode(&desc[idx], S_DIN_to_HASH);780 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);781 idx++;782 783 /* Load the hash current length*/784 hw_desc_init(&desc[idx]);785 set_cipher_mode(&desc[idx], ctx->hw_mode);786 set_din_const(&desc[idx], 0, ctx->hash_len);787 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);788 set_flow_mode(&desc[idx], S_DIN_to_HASH);789 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);790 idx++;791 792 hw_desc_init(&desc[idx]);793 set_din_type(&desc[idx], DMA_DLLI,794 ctx->key_params.key_dma_addr, keylen,795 NS_BIT);796 set_flow_mode(&desc[idx], DIN_HASH);797 idx++;798 799 /* Get hashed key */800 hw_desc_init(&desc[idx]);801 set_cipher_mode(&desc[idx], ctx->hw_mode);802 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,803 digestsize, NS_BIT, 0);804 set_flow_mode(&desc[idx], S_HASH_to_DOUT);805 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);806 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);807 cc_set_endianity(ctx->hash_mode, &desc[idx]);808 idx++;809 810 hw_desc_init(&desc[idx]);811 set_din_const(&desc[idx], 0, (blocksize - digestsize));812 set_flow_mode(&desc[idx], BYPASS);813 set_dout_dlli(&desc[idx],814 (ctx->opad_tmp_keys_dma_addr +815 digestsize),816 (blocksize - digestsize), NS_BIT, 0);817 idx++;818 } else {819 hw_desc_init(&desc[idx]);820 set_din_type(&desc[idx], DMA_DLLI,821 ctx->key_params.key_dma_addr, keylen,822 NS_BIT);823 set_flow_mode(&desc[idx], BYPASS);824 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,825 keylen, NS_BIT, 0);826 idx++;827 828 if ((blocksize - keylen)) {829 hw_desc_init(&desc[idx]);830 set_din_const(&desc[idx], 0,831 (blocksize - keylen));832 set_flow_mode(&desc[idx], BYPASS);833 set_dout_dlli(&desc[idx],834 (ctx->opad_tmp_keys_dma_addr +835 keylen), (blocksize - keylen),836 NS_BIT, 0);837 idx++;838 }839 }840 } else {841 hw_desc_init(&desc[idx]);842 set_din_const(&desc[idx], 0, blocksize);843 set_flow_mode(&desc[idx], BYPASS);844 set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr),845 blocksize, NS_BIT, 0);846 idx++;847 }848 849 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);850 if (rc) {851 dev_err(dev, "send_request() failed (rc=%d)\n", rc);852 goto out;853 }854 855 /* calc derived HMAC key */856 for (idx = 0, i = 0; i < 2; i++) {857 /* Load hash initial state */858 hw_desc_init(&desc[idx]);859 set_cipher_mode(&desc[idx], ctx->hw_mode);860 set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize);861 set_flow_mode(&desc[idx], S_DIN_to_HASH);862 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);863 idx++;864 865 /* Load the hash current length*/866 hw_desc_init(&desc[idx]);867 set_cipher_mode(&desc[idx], ctx->hw_mode);868 set_din_const(&desc[idx], 0, ctx->hash_len);869 set_flow_mode(&desc[idx], S_DIN_to_HASH);870 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);871 idx++;872 873 /* Prepare ipad key */874 hw_desc_init(&desc[idx]);875 set_xor_val(&desc[idx], hmac_pad_const[i]);876 set_cipher_mode(&desc[idx], ctx->hw_mode);877 set_flow_mode(&desc[idx], S_DIN_to_HASH);878 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);879 idx++;880 881 /* Perform HASH update */882 hw_desc_init(&desc[idx]);883 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,884 blocksize, NS_BIT);885 set_cipher_mode(&desc[idx], ctx->hw_mode);886 set_xor_active(&desc[idx]);887 set_flow_mode(&desc[idx], DIN_HASH);888 idx++;889 890 /* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest891 * of the first HASH "update" state)892 */893 hw_desc_init(&desc[idx]);894 set_cipher_mode(&desc[idx], ctx->hw_mode);895 if (i > 0) /* Not first iteration */896 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,897 ctx->inter_digestsize, NS_BIT, 0);898 else /* First iteration */899 set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr,900 ctx->inter_digestsize, NS_BIT, 0);901 set_flow_mode(&desc[idx], S_HASH_to_DOUT);902 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);903 idx++;904 }905 906 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);907 908out:909 if (ctx->key_params.key_dma_addr) {910 dma_unmap_single(dev, ctx->key_params.key_dma_addr,911 ctx->key_params.keylen, DMA_TO_DEVICE);912 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",913 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);914 }915 916 kfree_sensitive(ctx->key_params.key);917 918 return rc;919}920 921static int cc_xcbc_setkey(struct crypto_ahash *ahash,922 const u8 *key, unsigned int keylen)923{924 struct cc_crypto_req cc_req = {};925 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);926 struct device *dev = drvdata_to_dev(ctx->drvdata);927 int rc = 0;928 unsigned int idx = 0;929 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];930 931 dev_dbg(dev, "===== setkey (%d) ====\n", keylen);932 933 switch (keylen) {934 case AES_KEYSIZE_128:935 case AES_KEYSIZE_192:936 case AES_KEYSIZE_256:937 break;938 default:939 return -EINVAL;940 }941 942 ctx->key_params.keylen = keylen;943 944 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL);945 if (!ctx->key_params.key)946 return -ENOMEM;947 948 ctx->key_params.key_dma_addr =949 dma_map_single(dev, ctx->key_params.key, keylen, DMA_TO_DEVICE);950 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {951 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",952 key, keylen);953 kfree_sensitive(ctx->key_params.key);954 return -ENOMEM;955 }956 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",957 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);958 959 ctx->is_hmac = true;960 /* 1. Load the AES key */961 hw_desc_init(&desc[idx]);962 set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr,963 keylen, NS_BIT);964 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);965 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);966 set_key_size_aes(&desc[idx], keylen);967 set_flow_mode(&desc[idx], S_DIN_to_AES);968 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);969 idx++;970 971 hw_desc_init(&desc[idx]);972 set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE);973 set_flow_mode(&desc[idx], DIN_AES_DOUT);974 set_dout_dlli(&desc[idx],975 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),976 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);977 idx++;978 979 hw_desc_init(&desc[idx]);980 set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE);981 set_flow_mode(&desc[idx], DIN_AES_DOUT);982 set_dout_dlli(&desc[idx],983 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),984 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);985 idx++;986 987 hw_desc_init(&desc[idx]);988 set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE);989 set_flow_mode(&desc[idx], DIN_AES_DOUT);990 set_dout_dlli(&desc[idx],991 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),992 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);993 idx++;994 995 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);996 997 dma_unmap_single(dev, ctx->key_params.key_dma_addr,998 ctx->key_params.keylen, DMA_TO_DEVICE);999 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",1000 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);1001 1002 kfree_sensitive(ctx->key_params.key);1003 1004 return rc;1005}1006 1007static int cc_cmac_setkey(struct crypto_ahash *ahash,1008 const u8 *key, unsigned int keylen)1009{1010 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);1011 struct device *dev = drvdata_to_dev(ctx->drvdata);1012 1013 dev_dbg(dev, "===== setkey (%d) ====\n", keylen);1014 1015 ctx->is_hmac = true;1016 1017 switch (keylen) {1018 case AES_KEYSIZE_128:1019 case AES_KEYSIZE_192:1020 case AES_KEYSIZE_256:1021 break;1022 default:1023 return -EINVAL;1024 }1025 1026 ctx->key_params.keylen = keylen;1027 1028 /* STAT_PHASE_1: Copy key to ctx */1029 1030 dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr,1031 keylen, DMA_TO_DEVICE);1032 1033 memcpy(ctx->opad_tmp_keys_buff, key, keylen);1034 if (keylen == 24) {1035 memset(ctx->opad_tmp_keys_buff + 24, 0,1036 CC_AES_KEY_SIZE_MAX - 24);1037 }1038 1039 dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr,1040 keylen, DMA_TO_DEVICE);1041 1042 ctx->key_params.keylen = keylen;1043 1044 return 0;1045}1046 1047static void cc_free_ctx(struct cc_hash_ctx *ctx)1048{1049 struct device *dev = drvdata_to_dev(ctx->drvdata);1050 1051 if (ctx->digest_buff_dma_addr) {1052 dma_unmap_single(dev, ctx->digest_buff_dma_addr,1053 sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);1054 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",1055 &ctx->digest_buff_dma_addr);1056 ctx->digest_buff_dma_addr = 0;1057 }1058 if (ctx->opad_tmp_keys_dma_addr) {1059 dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr,1060 sizeof(ctx->opad_tmp_keys_buff),1061 DMA_BIDIRECTIONAL);1062 dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n",1063 &ctx->opad_tmp_keys_dma_addr);1064 ctx->opad_tmp_keys_dma_addr = 0;1065 }1066 1067 ctx->key_params.keylen = 0;1068}1069 1070static int cc_alloc_ctx(struct cc_hash_ctx *ctx)1071{1072 struct device *dev = drvdata_to_dev(ctx->drvdata);1073 1074 ctx->key_params.keylen = 0;1075 1076 ctx->digest_buff_dma_addr =1077 dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff),1078 DMA_BIDIRECTIONAL);1079 if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) {1080 dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n",1081 sizeof(ctx->digest_buff), ctx->digest_buff);1082 goto fail;1083 }1084 dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n",1085 sizeof(ctx->digest_buff), ctx->digest_buff,1086 &ctx->digest_buff_dma_addr);1087 1088 ctx->opad_tmp_keys_dma_addr =1089 dma_map_single(dev, ctx->opad_tmp_keys_buff,1090 sizeof(ctx->opad_tmp_keys_buff),1091 DMA_BIDIRECTIONAL);1092 if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) {1093 dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n",1094 sizeof(ctx->opad_tmp_keys_buff),1095 ctx->opad_tmp_keys_buff);1096 goto fail;1097 }1098 dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n",1099 sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,1100 &ctx->opad_tmp_keys_dma_addr);1101 1102 ctx->is_hmac = false;1103 return 0;1104 1105fail:1106 cc_free_ctx(ctx);1107 return -ENOMEM;1108}1109 1110static int cc_get_hash_len(struct crypto_tfm *tfm)1111{1112 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);1113 1114 if (ctx->hash_mode == DRV_HASH_SM3)1115 return CC_SM3_HASH_LEN_SIZE;1116 else1117 return cc_get_default_hash_len(ctx->drvdata);1118}1119 1120static int cc_cra_init(struct crypto_tfm *tfm)1121{1122 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);1123 struct hash_alg_common *hash_alg_common =1124 container_of(tfm->__crt_alg, struct hash_alg_common, base);1125 struct ahash_alg *ahash_alg =1126 container_of(hash_alg_common, struct ahash_alg, halg);1127 struct cc_hash_alg *cc_alg =1128 container_of(ahash_alg, struct cc_hash_alg, ahash_alg);1129 1130 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),1131 sizeof(struct ahash_req_ctx));1132 1133 ctx->hash_mode = cc_alg->hash_mode;1134 ctx->hw_mode = cc_alg->hw_mode;1135 ctx->inter_digestsize = cc_alg->inter_digestsize;1136 ctx->drvdata = cc_alg->drvdata;1137 ctx->hash_len = cc_get_hash_len(tfm);1138 return cc_alloc_ctx(ctx);1139}1140 1141static void cc_cra_exit(struct crypto_tfm *tfm)1142{1143 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);1144 struct device *dev = drvdata_to_dev(ctx->drvdata);1145 1146 dev_dbg(dev, "cc_cra_exit");1147 cc_free_ctx(ctx);1148}1149 1150static int cc_mac_update(struct ahash_request *req)1151{1152 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);1153 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1154 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);1155 struct device *dev = drvdata_to_dev(ctx->drvdata);1156 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);1157 struct cc_crypto_req cc_req = {};1158 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];1159 int rc;1160 u32 idx = 0;1161 gfp_t flags = cc_gfp_flags(&req->base);1162 1163 if (req->nbytes == 0) {1164 /* no real updates required */1165 return 0;1166 }1167 1168 state->xcbc_count++;1169 1170 rc = cc_map_hash_request_update(ctx->drvdata, state, req->src,1171 req->nbytes, block_size, flags);1172 if (rc) {1173 if (rc == 1) {1174 dev_dbg(dev, " data size not require HW update %x\n",1175 req->nbytes);1176 /* No hardware updates are required */1177 return 0;1178 }1179 dev_err(dev, "map_ahash_request_update() failed\n");1180 return -ENOMEM;1181 }1182 1183 if (cc_map_req(dev, state, ctx)) {1184 dev_err(dev, "map_ahash_source() failed\n");1185 return -EINVAL;1186 }1187 1188 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)1189 cc_setup_xcbc(req, desc, &idx);1190 else1191 cc_setup_cmac(req, desc, &idx);1192 1193 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx);1194 1195 /* store the hash digest result in context */1196 hw_desc_init(&desc[idx]);1197 set_cipher_mode(&desc[idx], ctx->hw_mode);1198 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,1199 ctx->inter_digestsize, NS_BIT, 1);1200 set_queue_last_ind(ctx->drvdata, &desc[idx]);1201 set_flow_mode(&desc[idx], S_AES_to_DOUT);1202 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);1203 idx++;1204 1205 /* Setup request structure */1206 cc_req.user_cb = cc_update_complete;1207 cc_req.user_arg = req;1208 1209 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);1210 if (rc != -EINPROGRESS && rc != -EBUSY) {1211 dev_err(dev, "send_request() failed (rc=%d)\n", rc);1212 cc_unmap_hash_request(dev, state, req->src, true);1213 cc_unmap_req(dev, state, ctx);1214 }1215 return rc;1216}1217 1218static int cc_mac_final(struct ahash_request *req)1219{1220 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);1221 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1222 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);1223 struct device *dev = drvdata_to_dev(ctx->drvdata);1224 struct cc_crypto_req cc_req = {};1225 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];1226 int idx = 0;1227 int rc = 0;1228 u32 key_size, key_len;1229 u32 digestsize = crypto_ahash_digestsize(tfm);1230 gfp_t flags = cc_gfp_flags(&req->base);1231 u32 rem_cnt = *cc_hash_buf_cnt(state);1232 1233 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {1234 key_size = CC_AES_128_BIT_KEY_SIZE;1235 key_len = CC_AES_128_BIT_KEY_SIZE;1236 } else {1237 key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :1238 ctx->key_params.keylen;1239 key_len = ctx->key_params.keylen;1240 }1241 1242 dev_dbg(dev, "===== final xcbc reminder (%d) ====\n", rem_cnt);1243 1244 if (cc_map_req(dev, state, ctx)) {1245 dev_err(dev, "map_ahash_source() failed\n");1246 return -EINVAL;1247 }1248 1249 if (cc_map_hash_request_final(ctx->drvdata, state, req->src,1250 req->nbytes, 0, flags)) {1251 dev_err(dev, "map_ahash_request_final() failed\n");1252 cc_unmap_req(dev, state, ctx);1253 return -ENOMEM;1254 }1255 1256 if (cc_map_result(dev, state, digestsize)) {1257 dev_err(dev, "map_ahash_digest() failed\n");1258 cc_unmap_hash_request(dev, state, req->src, true);1259 cc_unmap_req(dev, state, ctx);1260 return -ENOMEM;1261 }1262 1263 /* Setup request structure */1264 cc_req.user_cb = cc_hash_complete;1265 cc_req.user_arg = req;1266 1267 if (state->xcbc_count && rem_cnt == 0) {1268 /* Load key for ECB decryption */1269 hw_desc_init(&desc[idx]);1270 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);1271 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT);1272 set_din_type(&desc[idx], DMA_DLLI,1273 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),1274 key_size, NS_BIT);1275 set_key_size_aes(&desc[idx], key_len);1276 set_flow_mode(&desc[idx], S_DIN_to_AES);1277 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);1278 idx++;1279 1280 /* Initiate decryption of block state to previous1281 * block_state-XOR-M[n]1282 */1283 hw_desc_init(&desc[idx]);1284 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,1285 CC_AES_BLOCK_SIZE, NS_BIT);1286 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,1287 CC_AES_BLOCK_SIZE, NS_BIT, 0);1288 set_flow_mode(&desc[idx], DIN_AES_DOUT);1289 idx++;1290 1291 /* Memory Barrier: wait for axi write to complete */1292 hw_desc_init(&desc[idx]);1293 set_din_no_dma(&desc[idx], 0, 0xfffff0);1294 set_dout_no_dma(&desc[idx], 0, 0, 1);1295 idx++;1296 }1297 1298 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)1299 cc_setup_xcbc(req, desc, &idx);1300 else1301 cc_setup_cmac(req, desc, &idx);1302 1303 if (state->xcbc_count == 0) {1304 hw_desc_init(&desc[idx]);1305 set_cipher_mode(&desc[idx], ctx->hw_mode);1306 set_key_size_aes(&desc[idx], key_len);1307 set_cmac_size0_mode(&desc[idx]);1308 set_flow_mode(&desc[idx], S_DIN_to_AES);1309 idx++;1310 } else if (rem_cnt > 0) {1311 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);1312 } else {1313 hw_desc_init(&desc[idx]);1314 set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE);1315 set_flow_mode(&desc[idx], DIN_AES_DOUT);1316 idx++;1317 }1318 1319 /* Get final MAC result */1320 hw_desc_init(&desc[idx]);1321 set_dout_dlli(&desc[idx], state->digest_result_dma_addr,1322 digestsize, NS_BIT, 1);1323 set_queue_last_ind(ctx->drvdata, &desc[idx]);1324 set_flow_mode(&desc[idx], S_AES_to_DOUT);1325 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);1326 set_cipher_mode(&desc[idx], ctx->hw_mode);1327 idx++;1328 1329 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);1330 if (rc != -EINPROGRESS && rc != -EBUSY) {1331 dev_err(dev, "send_request() failed (rc=%d)\n", rc);1332 cc_unmap_hash_request(dev, state, req->src, true);1333 cc_unmap_result(dev, state, digestsize, req->result);1334 cc_unmap_req(dev, state, ctx);1335 }1336 return rc;1337}1338 1339static int cc_mac_finup(struct ahash_request *req)1340{1341 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);1342 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1343 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);1344 struct device *dev = drvdata_to_dev(ctx->drvdata);1345 struct cc_crypto_req cc_req = {};1346 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];1347 int idx = 0;1348 int rc = 0;1349 u32 key_len = 0;1350 u32 digestsize = crypto_ahash_digestsize(tfm);1351 gfp_t flags = cc_gfp_flags(&req->base);1352 1353 dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes);1354 if (state->xcbc_count > 0 && req->nbytes == 0) {1355 dev_dbg(dev, "No data to update. Call to fdx_mac_final\n");1356 return cc_mac_final(req);1357 }1358 1359 if (cc_map_req(dev, state, ctx)) {1360 dev_err(dev, "map_ahash_source() failed\n");1361 return -EINVAL;1362 }1363 1364 if (cc_map_hash_request_final(ctx->drvdata, state, req->src,1365 req->nbytes, 1, flags)) {1366 dev_err(dev, "map_ahash_request_final() failed\n");1367 cc_unmap_req(dev, state, ctx);1368 return -ENOMEM;1369 }1370 if (cc_map_result(dev, state, digestsize)) {1371 dev_err(dev, "map_ahash_digest() failed\n");1372 cc_unmap_hash_request(dev, state, req->src, true);1373 cc_unmap_req(dev, state, ctx);1374 return -ENOMEM;1375 }1376 1377 /* Setup request structure */1378 cc_req.user_cb = cc_hash_complete;1379 cc_req.user_arg = req;1380 1381 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {1382 key_len = CC_AES_128_BIT_KEY_SIZE;1383 cc_setup_xcbc(req, desc, &idx);1384 } else {1385 key_len = ctx->key_params.keylen;1386 cc_setup_cmac(req, desc, &idx);1387 }1388 1389 if (req->nbytes == 0) {1390 hw_desc_init(&desc[idx]);1391 set_cipher_mode(&desc[idx], ctx->hw_mode);1392 set_key_size_aes(&desc[idx], key_len);1393 set_cmac_size0_mode(&desc[idx]);1394 set_flow_mode(&desc[idx], S_DIN_to_AES);1395 idx++;1396 } else {1397 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);1398 }1399 1400 /* Get final MAC result */1401 hw_desc_init(&desc[idx]);1402 set_dout_dlli(&desc[idx], state->digest_result_dma_addr,1403 digestsize, NS_BIT, 1);1404 set_queue_last_ind(ctx->drvdata, &desc[idx]);1405 set_flow_mode(&desc[idx], S_AES_to_DOUT);1406 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);1407 set_cipher_mode(&desc[idx], ctx->hw_mode);1408 idx++;1409 1410 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);1411 if (rc != -EINPROGRESS && rc != -EBUSY) {1412 dev_err(dev, "send_request() failed (rc=%d)\n", rc);1413 cc_unmap_hash_request(dev, state, req->src, true);1414 cc_unmap_result(dev, state, digestsize, req->result);1415 cc_unmap_req(dev, state, ctx);1416 }1417 return rc;1418}1419 1420static int cc_mac_digest(struct ahash_request *req)1421{1422 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);1423 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1424 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);1425 struct device *dev = drvdata_to_dev(ctx->drvdata);1426 u32 digestsize = crypto_ahash_digestsize(tfm);1427 struct cc_crypto_req cc_req = {};1428 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];1429 u32 key_len;1430 unsigned int idx = 0;1431 int rc;1432 gfp_t flags = cc_gfp_flags(&req->base);1433 1434 dev_dbg(dev, "===== -digest mac (%d) ====\n", req->nbytes);1435 1436 cc_init_req(dev, state, ctx);1437 1438 if (cc_map_req(dev, state, ctx)) {1439 dev_err(dev, "map_ahash_source() failed\n");1440 return -ENOMEM;1441 }1442 if (cc_map_result(dev, state, digestsize)) {1443 dev_err(dev, "map_ahash_digest() failed\n");1444 cc_unmap_req(dev, state, ctx);1445 return -ENOMEM;1446 }1447 1448 if (cc_map_hash_request_final(ctx->drvdata, state, req->src,1449 req->nbytes, 1, flags)) {1450 dev_err(dev, "map_ahash_request_final() failed\n");1451 cc_unmap_req(dev, state, ctx);1452 return -ENOMEM;1453 }1454 1455 /* Setup request structure */1456 cc_req.user_cb = cc_digest_complete;1457 cc_req.user_arg = req;1458 1459 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {1460 key_len = CC_AES_128_BIT_KEY_SIZE;1461 cc_setup_xcbc(req, desc, &idx);1462 } else {1463 key_len = ctx->key_params.keylen;1464 cc_setup_cmac(req, desc, &idx);1465 }1466 1467 if (req->nbytes == 0) {1468 hw_desc_init(&desc[idx]);1469 set_cipher_mode(&desc[idx], ctx->hw_mode);1470 set_key_size_aes(&desc[idx], key_len);1471 set_cmac_size0_mode(&desc[idx]);1472 set_flow_mode(&desc[idx], S_DIN_to_AES);1473 idx++;1474 } else {1475 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);1476 }1477 1478 /* Get final MAC result */1479 hw_desc_init(&desc[idx]);1480 set_dout_dlli(&desc[idx], state->digest_result_dma_addr,1481 CC_AES_BLOCK_SIZE, NS_BIT, 1);1482 set_queue_last_ind(ctx->drvdata, &desc[idx]);1483 set_flow_mode(&desc[idx], S_AES_to_DOUT);1484 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);1485 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);1486 set_cipher_mode(&desc[idx], ctx->hw_mode);1487 idx++;1488 1489 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);1490 if (rc != -EINPROGRESS && rc != -EBUSY) {1491 dev_err(dev, "send_request() failed (rc=%d)\n", rc);1492 cc_unmap_hash_request(dev, state, req->src, true);1493 cc_unmap_result(dev, state, digestsize, req->result);1494 cc_unmap_req(dev, state, ctx);1495 }1496 return rc;1497}1498 1499static int cc_hash_export(struct ahash_request *req, void *out)1500{1501 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);1502 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);1503 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);1504 u8 *curr_buff = cc_hash_buf(state);1505 u32 curr_buff_cnt = *cc_hash_buf_cnt(state);1506 const u32 tmp = CC_EXPORT_MAGIC;1507 1508 memcpy(out, &tmp, sizeof(u32));1509 out += sizeof(u32);1510 1511 memcpy(out, state->digest_buff, ctx->inter_digestsize);1512 out += ctx->inter_digestsize;1513 1514 memcpy(out, state->digest_bytes_len, ctx->hash_len);1515 out += ctx->hash_len;1516 1517 memcpy(out, &curr_buff_cnt, sizeof(u32));1518 out += sizeof(u32);1519 1520 memcpy(out, curr_buff, curr_buff_cnt);1521 1522 return 0;1523}1524 1525static int cc_hash_import(struct ahash_request *req, const void *in)1526{1527 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);1528 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);1529 struct device *dev = drvdata_to_dev(ctx->drvdata);1530 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);1531 u32 tmp;1532 1533 memcpy(&tmp, in, sizeof(u32));1534 if (tmp != CC_EXPORT_MAGIC)1535 return -EINVAL;1536 in += sizeof(u32);1537 1538 cc_init_req(dev, state, ctx);1539 1540 memcpy(state->digest_buff, in, ctx->inter_digestsize);1541 in += ctx->inter_digestsize;1542 1543 memcpy(state->digest_bytes_len, in, ctx->hash_len);1544 in += ctx->hash_len;1545 1546 /* Sanity check the data as much as possible */1547 memcpy(&tmp, in, sizeof(u32));1548 if (tmp > CC_MAX_HASH_BLCK_SIZE)1549 return -EINVAL;1550 in += sizeof(u32);1551 1552 state->buf_cnt[0] = tmp;1553 memcpy(state->buffers[0], in, tmp);1554 1555 return 0;1556}1557 1558struct cc_hash_template {1559 char name[CRYPTO_MAX_ALG_NAME];1560 char driver_name[CRYPTO_MAX_ALG_NAME];1561 char mac_name[CRYPTO_MAX_ALG_NAME];1562 char mac_driver_name[CRYPTO_MAX_ALG_NAME];1563 unsigned int blocksize;1564 bool is_mac;1565 bool synchronize;1566 struct ahash_alg template_ahash;1567 int hash_mode;1568 int hw_mode;1569 int inter_digestsize;1570 struct cc_drvdata *drvdata;1571 u32 min_hw_rev;1572 enum cc_std_body std_body;1573};1574 1575#define CC_STATE_SIZE(_x) \1576 ((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32)))1577 1578/* hash descriptors */1579static struct cc_hash_template driver_hash[] = {1580 //Asynchronize hash template1581 {1582 .name = "sha1",1583 .driver_name = "sha1-ccree",1584 .mac_name = "hmac(sha1)",1585 .mac_driver_name = "hmac-sha1-ccree",1586 .blocksize = SHA1_BLOCK_SIZE,1587 .is_mac = true,1588 .synchronize = false,1589 .template_ahash = {1590 .init = cc_hash_init,1591 .update = cc_hash_update,1592 .final = cc_hash_final,1593 .finup = cc_hash_finup,1594 .digest = cc_hash_digest,1595 .export = cc_hash_export,1596 .import = cc_hash_import,1597 .setkey = cc_hash_setkey,1598 .halg = {1599 .digestsize = SHA1_DIGEST_SIZE,1600 .statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE),1601 },1602 },1603 .hash_mode = DRV_HASH_SHA1,1604 .hw_mode = DRV_HASH_HW_SHA1,1605 .inter_digestsize = SHA1_DIGEST_SIZE,1606 .min_hw_rev = CC_HW_REV_630,1607 .std_body = CC_STD_NIST,1608 },1609 {1610 .name = "sha256",1611 .driver_name = "sha256-ccree",1612 .mac_name = "hmac(sha256)",1613 .mac_driver_name = "hmac-sha256-ccree",1614 .blocksize = SHA256_BLOCK_SIZE,1615 .is_mac = true,1616 .template_ahash = {1617 .init = cc_hash_init,1618 .update = cc_hash_update,1619 .final = cc_hash_final,1620 .finup = cc_hash_finup,1621 .digest = cc_hash_digest,1622 .export = cc_hash_export,1623 .import = cc_hash_import,1624 .setkey = cc_hash_setkey,1625 .halg = {1626 .digestsize = SHA256_DIGEST_SIZE,1627 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE)1628 },1629 },1630 .hash_mode = DRV_HASH_SHA256,1631 .hw_mode = DRV_HASH_HW_SHA256,1632 .inter_digestsize = SHA256_DIGEST_SIZE,1633 .min_hw_rev = CC_HW_REV_630,1634 .std_body = CC_STD_NIST,1635 },1636 {1637 .name = "sha224",1638 .driver_name = "sha224-ccree",1639 .mac_name = "hmac(sha224)",1640 .mac_driver_name = "hmac-sha224-ccree",1641 .blocksize = SHA224_BLOCK_SIZE,1642 .is_mac = true,1643 .template_ahash = {1644 .init = cc_hash_init,1645 .update = cc_hash_update,1646 .final = cc_hash_final,1647 .finup = cc_hash_finup,1648 .digest = cc_hash_digest,1649 .export = cc_hash_export,1650 .import = cc_hash_import,1651 .setkey = cc_hash_setkey,1652 .halg = {1653 .digestsize = SHA224_DIGEST_SIZE,1654 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE),1655 },1656 },1657 .hash_mode = DRV_HASH_SHA224,1658 .hw_mode = DRV_HASH_HW_SHA256,1659 .inter_digestsize = SHA256_DIGEST_SIZE,1660 .min_hw_rev = CC_HW_REV_630,1661 .std_body = CC_STD_NIST,1662 },1663 {1664 .name = "sha384",1665 .driver_name = "sha384-ccree",1666 .mac_name = "hmac(sha384)",1667 .mac_driver_name = "hmac-sha384-ccree",1668 .blocksize = SHA384_BLOCK_SIZE,1669 .is_mac = true,1670 .template_ahash = {1671 .init = cc_hash_init,1672 .update = cc_hash_update,1673 .final = cc_hash_final,1674 .finup = cc_hash_finup,1675 .digest = cc_hash_digest,1676 .export = cc_hash_export,1677 .import = cc_hash_import,1678 .setkey = cc_hash_setkey,1679 .halg = {1680 .digestsize = SHA384_DIGEST_SIZE,1681 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE),1682 },1683 },1684 .hash_mode = DRV_HASH_SHA384,1685 .hw_mode = DRV_HASH_HW_SHA512,1686 .inter_digestsize = SHA512_DIGEST_SIZE,1687 .min_hw_rev = CC_HW_REV_712,1688 .std_body = CC_STD_NIST,1689 },1690 {1691 .name = "sha512",1692 .driver_name = "sha512-ccree",1693 .mac_name = "hmac(sha512)",1694 .mac_driver_name = "hmac-sha512-ccree",1695 .blocksize = SHA512_BLOCK_SIZE,1696 .is_mac = true,1697 .template_ahash = {1698 .init = cc_hash_init,1699 .update = cc_hash_update,1700 .final = cc_hash_final,1701 .finup = cc_hash_finup,1702 .digest = cc_hash_digest,1703 .export = cc_hash_export,1704 .import = cc_hash_import,1705 .setkey = cc_hash_setkey,1706 .halg = {1707 .digestsize = SHA512_DIGEST_SIZE,1708 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE),1709 },1710 },1711 .hash_mode = DRV_HASH_SHA512,1712 .hw_mode = DRV_HASH_HW_SHA512,1713 .inter_digestsize = SHA512_DIGEST_SIZE,1714 .min_hw_rev = CC_HW_REV_712,1715 .std_body = CC_STD_NIST,1716 },1717 {1718 .name = "md5",1719 .driver_name = "md5-ccree",1720 .mac_name = "hmac(md5)",1721 .mac_driver_name = "hmac-md5-ccree",1722 .blocksize = MD5_HMAC_BLOCK_SIZE,1723 .is_mac = true,1724 .template_ahash = {1725 .init = cc_hash_init,1726 .update = cc_hash_update,1727 .final = cc_hash_final,1728 .finup = cc_hash_finup,1729 .digest = cc_hash_digest,1730 .export = cc_hash_export,1731 .import = cc_hash_import,1732 .setkey = cc_hash_setkey,1733 .halg = {1734 .digestsize = MD5_DIGEST_SIZE,1735 .statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE),1736 },1737 },1738 .hash_mode = DRV_HASH_MD5,1739 .hw_mode = DRV_HASH_HW_MD5,1740 .inter_digestsize = MD5_DIGEST_SIZE,1741 .min_hw_rev = CC_HW_REV_630,1742 .std_body = CC_STD_NIST,1743 },1744 {1745 .name = "sm3",1746 .driver_name = "sm3-ccree",1747 .blocksize = SM3_BLOCK_SIZE,1748 .is_mac = false,1749 .template_ahash = {1750 .init = cc_hash_init,1751 .update = cc_hash_update,1752 .final = cc_hash_final,1753 .finup = cc_hash_finup,1754 .digest = cc_hash_digest,1755 .export = cc_hash_export,1756 .import = cc_hash_import,1757 .setkey = cc_hash_setkey,1758 .halg = {1759 .digestsize = SM3_DIGEST_SIZE,1760 .statesize = CC_STATE_SIZE(SM3_DIGEST_SIZE),1761 },1762 },1763 .hash_mode = DRV_HASH_SM3,1764 .hw_mode = DRV_HASH_HW_SM3,1765 .inter_digestsize = SM3_DIGEST_SIZE,1766 .min_hw_rev = CC_HW_REV_713,1767 .std_body = CC_STD_OSCCA,1768 },1769 {1770 .mac_name = "xcbc(aes)",1771 .mac_driver_name = "xcbc-aes-ccree",1772 .blocksize = AES_BLOCK_SIZE,1773 .is_mac = true,1774 .template_ahash = {1775 .init = cc_hash_init,1776 .update = cc_mac_update,1777 .final = cc_mac_final,1778 .finup = cc_mac_finup,1779 .digest = cc_mac_digest,1780 .setkey = cc_xcbc_setkey,1781 .export = cc_hash_export,1782 .import = cc_hash_import,1783 .halg = {1784 .digestsize = AES_BLOCK_SIZE,1785 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),1786 },1787 },1788 .hash_mode = DRV_HASH_NULL,1789 .hw_mode = DRV_CIPHER_XCBC_MAC,1790 .inter_digestsize = AES_BLOCK_SIZE,1791 .min_hw_rev = CC_HW_REV_630,1792 .std_body = CC_STD_NIST,1793 },1794 {1795 .mac_name = "cmac(aes)",1796 .mac_driver_name = "cmac-aes-ccree",1797 .blocksize = AES_BLOCK_SIZE,1798 .is_mac = true,1799 .template_ahash = {1800 .init = cc_hash_init,1801 .update = cc_mac_update,1802 .final = cc_mac_final,1803 .finup = cc_mac_finup,1804 .digest = cc_mac_digest,1805 .setkey = cc_cmac_setkey,1806 .export = cc_hash_export,1807 .import = cc_hash_import,1808 .halg = {1809 .digestsize = AES_BLOCK_SIZE,1810 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),1811 },1812 },1813 .hash_mode = DRV_HASH_NULL,1814 .hw_mode = DRV_CIPHER_CMAC,1815 .inter_digestsize = AES_BLOCK_SIZE,1816 .min_hw_rev = CC_HW_REV_630,1817 .std_body = CC_STD_NIST,1818 },1819};1820 1821static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,1822 struct device *dev, bool keyed)1823{1824 struct cc_hash_alg *t_crypto_alg;1825 struct crypto_alg *alg;1826 struct ahash_alg *halg;1827 1828 t_crypto_alg = devm_kzalloc(dev, sizeof(*t_crypto_alg), GFP_KERNEL);1829 if (!t_crypto_alg)1830 return ERR_PTR(-ENOMEM);1831 1832 t_crypto_alg->ahash_alg = template->template_ahash;1833 halg = &t_crypto_alg->ahash_alg;1834 alg = &halg->halg.base;1835 1836 if (keyed) {1837 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",1838 template->mac_name);1839 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",1840 template->mac_driver_name);1841 } else {1842 halg->setkey = NULL;1843 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",1844 template->name);1845 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",1846 template->driver_name);1847 }1848 alg->cra_module = THIS_MODULE;1849 alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding();1850 alg->cra_priority = CC_CRA_PRIO;1851 alg->cra_blocksize = template->blocksize;1852 alg->cra_alignmask = 0;1853 alg->cra_exit = cc_cra_exit;1854 1855 alg->cra_init = cc_cra_init;1856 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;1857 1858 t_crypto_alg->hash_mode = template->hash_mode;1859 t_crypto_alg->hw_mode = template->hw_mode;1860 t_crypto_alg->inter_digestsize = template->inter_digestsize;1861 1862 return t_crypto_alg;1863}1864 1865static int cc_init_copy_sram(struct cc_drvdata *drvdata, const u32 *data,1866 unsigned int size, u32 *sram_buff_ofs)1867{1868 struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)];1869 unsigned int larval_seq_len = 0;1870 int rc;1871 1872 cc_set_sram_desc(data, *sram_buff_ofs, size / sizeof(*data),1873 larval_seq, &larval_seq_len);1874 rc = send_request_init(drvdata, larval_seq, larval_seq_len);1875 if (rc)1876 return rc;1877 1878 *sram_buff_ofs += size;1879 return 0;1880}1881 1882int cc_init_hash_sram(struct cc_drvdata *drvdata)1883{1884 struct cc_hash_handle *hash_handle = drvdata->hash_handle;1885 u32 sram_buff_ofs = hash_handle->digest_len_sram_addr;1886 bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712);1887 bool sm3_supported = (drvdata->hw_rev >= CC_HW_REV_713);1888 int rc = 0;1889 1890 /* Copy-to-sram digest-len */1891 rc = cc_init_copy_sram(drvdata, cc_digest_len_init,1892 sizeof(cc_digest_len_init), &sram_buff_ofs);1893 if (rc)1894 goto init_digest_const_err;1895 1896 if (large_sha_supported) {1897 /* Copy-to-sram digest-len for sha384/512 */1898 rc = cc_init_copy_sram(drvdata, cc_digest_len_sha512_init,1899 sizeof(cc_digest_len_sha512_init),1900 &sram_buff_ofs);1901 if (rc)1902 goto init_digest_const_err;1903 }1904 1905 /* The initial digests offset */1906 hash_handle->larval_digest_sram_addr = sram_buff_ofs;1907 1908 /* Copy-to-sram initial SHA* digests */1909 rc = cc_init_copy_sram(drvdata, cc_md5_init, sizeof(cc_md5_init),1910 &sram_buff_ofs);1911 if (rc)1912 goto init_digest_const_err;1913 1914 rc = cc_init_copy_sram(drvdata, cc_sha1_init, sizeof(cc_sha1_init),1915 &sram_buff_ofs);1916 if (rc)1917 goto init_digest_const_err;1918 1919 rc = cc_init_copy_sram(drvdata, cc_sha224_init, sizeof(cc_sha224_init),1920 &sram_buff_ofs);1921 if (rc)1922 goto init_digest_const_err;1923 1924 rc = cc_init_copy_sram(drvdata, cc_sha256_init, sizeof(cc_sha256_init),1925 &sram_buff_ofs);1926 if (rc)1927 goto init_digest_const_err;1928 1929 if (sm3_supported) {1930 rc = cc_init_copy_sram(drvdata, cc_sm3_init,1931 sizeof(cc_sm3_init), &sram_buff_ofs);1932 if (rc)1933 goto init_digest_const_err;1934 }1935 1936 if (large_sha_supported) {1937 rc = cc_init_copy_sram(drvdata, cc_sha384_init,1938 sizeof(cc_sha384_init), &sram_buff_ofs);1939 if (rc)1940 goto init_digest_const_err;1941 1942 rc = cc_init_copy_sram(drvdata, cc_sha512_init,1943 sizeof(cc_sha512_init), &sram_buff_ofs);1944 if (rc)1945 goto init_digest_const_err;1946 }1947 1948init_digest_const_err:1949 return rc;1950}1951 1952int cc_hash_alloc(struct cc_drvdata *drvdata)1953{1954 struct cc_hash_handle *hash_handle;1955 u32 sram_buff;1956 u32 sram_size_to_alloc;1957 struct device *dev = drvdata_to_dev(drvdata);1958 int rc = 0;1959 int alg;1960 1961 hash_handle = devm_kzalloc(dev, sizeof(*hash_handle), GFP_KERNEL);1962 if (!hash_handle)1963 return -ENOMEM;1964 1965 INIT_LIST_HEAD(&hash_handle->hash_list);1966 drvdata->hash_handle = hash_handle;1967 1968 sram_size_to_alloc = sizeof(cc_digest_len_init) +1969 sizeof(cc_md5_init) +1970 sizeof(cc_sha1_init) +1971 sizeof(cc_sha224_init) +1972 sizeof(cc_sha256_init);1973 1974 if (drvdata->hw_rev >= CC_HW_REV_713)1975 sram_size_to_alloc += sizeof(cc_sm3_init);1976 1977 if (drvdata->hw_rev >= CC_HW_REV_712)1978 sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) +1979 sizeof(cc_sha384_init) + sizeof(cc_sha512_init);1980 1981 sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc);1982 if (sram_buff == NULL_SRAM_ADDR) {1983 rc = -ENOMEM;1984 goto fail;1985 }1986 1987 /* The initial digest-len offset */1988 hash_handle->digest_len_sram_addr = sram_buff;1989 1990 /*must be set before the alg registration as it is being used there*/1991 rc = cc_init_hash_sram(drvdata);1992 if (rc) {1993 dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc);1994 goto fail;1995 }1996 1997 /* ahash registration */1998 for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) {1999 struct cc_hash_alg *t_alg;2000 int hw_mode = driver_hash[alg].hw_mode;2001 2002 /* Check that the HW revision and variants are suitable */2003 if ((driver_hash[alg].min_hw_rev > drvdata->hw_rev) ||2004 !(drvdata->std_bodies & driver_hash[alg].std_body))2005 continue;2006 2007 if (driver_hash[alg].is_mac) {2008 /* register hmac version */2009 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true);2010 if (IS_ERR(t_alg)) {2011 rc = PTR_ERR(t_alg);2012 dev_err(dev, "%s alg allocation failed\n",2013 driver_hash[alg].driver_name);2014 goto fail;2015 }2016 t_alg->drvdata = drvdata;2017 2018 rc = crypto_register_ahash(&t_alg->ahash_alg);2019 if (rc) {2020 dev_err(dev, "%s alg registration failed\n",2021 driver_hash[alg].driver_name);2022 goto fail;2023 }2024 2025 list_add_tail(&t_alg->entry, &hash_handle->hash_list);2026 }2027 if (hw_mode == DRV_CIPHER_XCBC_MAC ||2028 hw_mode == DRV_CIPHER_CMAC)2029 continue;2030 2031 /* register hash version */2032 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false);2033 if (IS_ERR(t_alg)) {2034 rc = PTR_ERR(t_alg);2035 dev_err(dev, "%s alg allocation failed\n",2036 driver_hash[alg].driver_name);2037 goto fail;2038 }2039 t_alg->drvdata = drvdata;2040 2041 rc = crypto_register_ahash(&t_alg->ahash_alg);2042 if (rc) {2043 dev_err(dev, "%s alg registration failed\n",2044 driver_hash[alg].driver_name);2045 goto fail;2046 }2047 2048 list_add_tail(&t_alg->entry, &hash_handle->hash_list);2049 }2050 2051 return 0;2052 2053fail:2054 cc_hash_free(drvdata);2055 return rc;2056}2057 2058int cc_hash_free(struct cc_drvdata *drvdata)2059{2060 struct cc_hash_alg *t_hash_alg, *hash_n;2061 struct cc_hash_handle *hash_handle = drvdata->hash_handle;2062 2063 list_for_each_entry_safe(t_hash_alg, hash_n, &hash_handle->hash_list,2064 entry) {2065 crypto_unregister_ahash(&t_hash_alg->ahash_alg);2066 list_del(&t_hash_alg->entry);2067 }2068 2069 return 0;2070}2071 2072static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],2073 unsigned int *seq_size)2074{2075 unsigned int idx = *seq_size;2076 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq);2077 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);2078 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);2079 2080 /* Setup XCBC MAC K1 */2081 hw_desc_init(&desc[idx]);2082 set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr +2083 XCBC_MAC_K1_OFFSET),2084 CC_AES_128_BIT_KEY_SIZE, NS_BIT);2085 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);2086 set_hash_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC, ctx->hash_mode);2087 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);2088 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);2089 set_flow_mode(&desc[idx], S_DIN_to_AES);2090 idx++;2091 2092 /* Setup XCBC MAC K2 */2093 hw_desc_init(&desc[idx]);2094 set_din_type(&desc[idx], DMA_DLLI,2095 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),2096 CC_AES_128_BIT_KEY_SIZE, NS_BIT);2097 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);2098 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);2099 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);2100 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);2101 set_flow_mode(&desc[idx], S_DIN_to_AES);2102 idx++;2103 2104 /* Setup XCBC MAC K3 */2105 hw_desc_init(&desc[idx]);2106 set_din_type(&desc[idx], DMA_DLLI,2107 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),2108 CC_AES_128_BIT_KEY_SIZE, NS_BIT);2109 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);2110 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);2111 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);2112 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);2113 set_flow_mode(&desc[idx], S_DIN_to_AES);2114 idx++;2115 2116 /* Loading MAC state */2117 hw_desc_init(&desc[idx]);2118 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,2119 CC_AES_BLOCK_SIZE, NS_BIT);2120 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);2121 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);2122 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);2123 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);2124 set_flow_mode(&desc[idx], S_DIN_to_AES);2125 idx++;2126 *seq_size = idx;2127}2128 2129static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],2130 unsigned int *seq_size)2131{2132 unsigned int idx = *seq_size;2133 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq);2134 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);2135 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);2136 2137 /* Setup CMAC Key */2138 hw_desc_init(&desc[idx]);2139 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,2140 ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :2141 ctx->key_params.keylen), NS_BIT);2142 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);2143 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);2144 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);2145 set_key_size_aes(&desc[idx], ctx->key_params.keylen);2146 set_flow_mode(&desc[idx], S_DIN_to_AES);2147 idx++;2148 2149 /* Load MAC state */2150 hw_desc_init(&desc[idx]);2151 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,2152 CC_AES_BLOCK_SIZE, NS_BIT);2153 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);2154 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);2155 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);2156 set_key_size_aes(&desc[idx], ctx->key_params.keylen);2157 set_flow_mode(&desc[idx], S_DIN_to_AES);2158 idx++;2159 *seq_size = idx;2160}2161 2162static void cc_set_desc(struct ahash_req_ctx *areq_ctx,2163 struct cc_hash_ctx *ctx, unsigned int flow_mode,2164 struct cc_hw_desc desc[], bool is_not_last_data,2165 unsigned int *seq_size)2166{2167 unsigned int idx = *seq_size;2168 struct device *dev = drvdata_to_dev(ctx->drvdata);2169 2170 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) {2171 hw_desc_init(&desc[idx]);2172 set_din_type(&desc[idx], DMA_DLLI,2173 sg_dma_address(areq_ctx->curr_sg),2174 areq_ctx->curr_sg->length, NS_BIT);2175 set_flow_mode(&desc[idx], flow_mode);2176 idx++;2177 } else {2178 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {2179 dev_dbg(dev, " NULL mode\n");2180 /* nothing to build */2181 return;2182 }2183 /* bypass */2184 hw_desc_init(&desc[idx]);2185 set_din_type(&desc[idx], DMA_DLLI,2186 areq_ctx->mlli_params.mlli_dma_addr,2187 areq_ctx->mlli_params.mlli_len, NS_BIT);2188 set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr,2189 areq_ctx->mlli_params.mlli_len);2190 set_flow_mode(&desc[idx], BYPASS);2191 idx++;2192 /* process */2193 hw_desc_init(&desc[idx]);2194 set_din_type(&desc[idx], DMA_MLLI,2195 ctx->drvdata->mlli_sram_addr,2196 areq_ctx->mlli_nents, NS_BIT);2197 set_flow_mode(&desc[idx], flow_mode);2198 idx++;2199 }2200 if (is_not_last_data)2201 set_din_not_last_indication(&desc[(idx - 1)]);2202 /* return updated desc sequence size */2203 *seq_size = idx;2204}2205 2206static const void *cc_larval_digest(struct device *dev, u32 mode)2207{2208 switch (mode) {2209 case DRV_HASH_MD5:2210 return cc_md5_init;2211 case DRV_HASH_SHA1:2212 return cc_sha1_init;2213 case DRV_HASH_SHA224:2214 return cc_sha224_init;2215 case DRV_HASH_SHA256:2216 return cc_sha256_init;2217 case DRV_HASH_SHA384:2218 return cc_sha384_init;2219 case DRV_HASH_SHA512:2220 return cc_sha512_init;2221 case DRV_HASH_SM3:2222 return cc_sm3_init;2223 default:2224 dev_err(dev, "Invalid hash mode (%d)\n", mode);2225 return cc_md5_init;2226 }2227}2228 2229/**2230 * cc_larval_digest_addr() - Get the address of the initial digest in SRAM2231 * according to the given hash mode2232 *2233 * @drvdata: Associated device driver context2234 * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA2562235 *2236 * Return:2237 * The address of the initial digest in SRAM2238 */2239u32 cc_larval_digest_addr(void *drvdata, u32 mode)2240{2241 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;2242 struct cc_hash_handle *hash_handle = _drvdata->hash_handle;2243 struct device *dev = drvdata_to_dev(_drvdata);2244 bool sm3_supported = (_drvdata->hw_rev >= CC_HW_REV_713);2245 u32 addr;2246 2247 switch (mode) {2248 case DRV_HASH_NULL:2249 break; /*Ignore*/2250 case DRV_HASH_MD5:2251 return (hash_handle->larval_digest_sram_addr);2252 case DRV_HASH_SHA1:2253 return (hash_handle->larval_digest_sram_addr +2254 sizeof(cc_md5_init));2255 case DRV_HASH_SHA224:2256 return (hash_handle->larval_digest_sram_addr +2257 sizeof(cc_md5_init) +2258 sizeof(cc_sha1_init));2259 case DRV_HASH_SHA256:2260 return (hash_handle->larval_digest_sram_addr +2261 sizeof(cc_md5_init) +2262 sizeof(cc_sha1_init) +2263 sizeof(cc_sha224_init));2264 case DRV_HASH_SM3:2265 return (hash_handle->larval_digest_sram_addr +2266 sizeof(cc_md5_init) +2267 sizeof(cc_sha1_init) +2268 sizeof(cc_sha224_init) +2269 sizeof(cc_sha256_init));2270 case DRV_HASH_SHA384:2271 addr = (hash_handle->larval_digest_sram_addr +2272 sizeof(cc_md5_init) +2273 sizeof(cc_sha1_init) +2274 sizeof(cc_sha224_init) +2275 sizeof(cc_sha256_init));2276 if (sm3_supported)2277 addr += sizeof(cc_sm3_init);2278 return addr;2279 case DRV_HASH_SHA512:2280 addr = (hash_handle->larval_digest_sram_addr +2281 sizeof(cc_md5_init) +2282 sizeof(cc_sha1_init) +2283 sizeof(cc_sha224_init) +2284 sizeof(cc_sha256_init) +2285 sizeof(cc_sha384_init));2286 if (sm3_supported)2287 addr += sizeof(cc_sm3_init);2288 return addr;2289 default:2290 dev_err(dev, "Invalid hash mode (%d)\n", mode);2291 }2292 2293 /*This is valid wrong value to avoid kernel crash*/2294 return hash_handle->larval_digest_sram_addr;2295}2296 2297u32 cc_digest_len_addr(void *drvdata, u32 mode)2298{2299 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;2300 struct cc_hash_handle *hash_handle = _drvdata->hash_handle;2301 u32 digest_len_addr = hash_handle->digest_len_sram_addr;2302 2303 switch (mode) {2304 case DRV_HASH_SHA1:2305 case DRV_HASH_SHA224:2306 case DRV_HASH_SHA256:2307 case DRV_HASH_MD5:2308 return digest_len_addr;2309 case DRV_HASH_SHA384:2310 case DRV_HASH_SHA512:2311 return digest_len_addr + sizeof(cc_digest_len_init);2312 default:2313 return digest_len_addr; /*to avoid kernel crash*/2314 }2315}2316