1690 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Intel Keem Bay OCS AES Crypto Driver.4 *5 * Copyright (C) 2018-2020 Intel Corporation6 */7 8#include <crypto/aes.h>9#include <crypto/engine.h>10#include <crypto/gcm.h>11#include <crypto/internal/aead.h>12#include <crypto/internal/skcipher.h>13#include <crypto/scatterwalk.h>14#include <linux/clk.h>15#include <linux/completion.h>16#include <linux/dma-mapping.h>17#include <linux/err.h>18#include <linux/interrupt.h>19#include <linux/io.h>20#include <linux/kernel.h>21#include <linux/module.h>22#include <linux/of.h>23#include <linux/platform_device.h>24#include <linux/string.h>25 26#include "ocs-aes.h"27 28#define KMB_OCS_PRIORITY 35029#define DRV_NAME "keembay-ocs-aes"30 31#define OCS_AES_MIN_KEY_SIZE 1632#define OCS_AES_MAX_KEY_SIZE 3233#define OCS_AES_KEYSIZE_128 1634#define OCS_AES_KEYSIZE_192 2435#define OCS_AES_KEYSIZE_256 3236#define OCS_SM4_KEY_SIZE 1637 38/**39 * struct ocs_aes_tctx - OCS AES Transform context40 * @aes_dev: The OCS AES device.41 * @key: AES/SM4 key.42 * @key_len: The length (in bytes) of @key.43 * @cipher: OCS cipher to use (either AES or SM4).44 * @sw_cipher: The cipher to use as fallback.45 * @use_fallback: Whether or not fallback cipher should be used.46 */47struct ocs_aes_tctx {48 struct ocs_aes_dev *aes_dev;49 u8 key[OCS_AES_KEYSIZE_256];50 unsigned int key_len;51 enum ocs_cipher cipher;52 union {53 struct crypto_sync_skcipher *sk;54 struct crypto_aead *aead;55 } sw_cipher;56 bool use_fallback;57};58 59/**60 * struct ocs_aes_rctx - OCS AES Request context.61 * @instruction: Instruction to be executed (encrypt / decrypt).62 * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS)63 * @src_nents: Number of source SG entries.64 * @dst_nents: Number of destination SG entries.65 * @src_dma_count: The number of DMA-mapped entries of the source SG.66 * @dst_dma_count: The number of DMA-mapped entries of the destination SG.67 * @in_place: Whether or not this is an in place request, i.e.,68 * src_sg == dst_sg.69 * @src_dll: OCS DMA linked list for input data.70 * @dst_dll: OCS DMA linked list for output data.71 * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC72 * mode).73 * @cts_swap: Whether or not CTS swap must be performed.74 * @aad_src_dll: OCS DMA linked list for input AAD data.75 * @aad_dst_dll: OCS DMA linked list for output AAD data.76 * @in_tag: Buffer to hold input encrypted tag (only used for77 * CCM/GCM decrypt).78 * @out_tag: Buffer to hold output encrypted / decrypted tag (only79 * used for GCM encrypt / decrypt).80 */81struct ocs_aes_rctx {82 /* Fields common across all modes. */83 enum ocs_instruction instruction;84 enum ocs_mode mode;85 int src_nents;86 int dst_nents;87 int src_dma_count;88 int dst_dma_count;89 bool in_place;90 struct ocs_dll_desc src_dll;91 struct ocs_dll_desc dst_dll;92 93 /* CBC specific */94 u8 last_ct_blk[AES_BLOCK_SIZE];95 96 /* CTS specific */97 int cts_swap;98 99 /* CCM/GCM specific */100 struct ocs_dll_desc aad_src_dll;101 struct ocs_dll_desc aad_dst_dll;102 u8 in_tag[AES_BLOCK_SIZE];103 104 /* GCM specific */105 u8 out_tag[AES_BLOCK_SIZE];106};107 108/* Driver data. */109struct ocs_aes_drv {110 struct list_head dev_list;111 spinlock_t lock; /* Protects dev_list. */112};113 114static struct ocs_aes_drv ocs_aes = {115 .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list),116 .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock),117};118 119static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx)120{121 struct ocs_aes_dev *aes_dev;122 123 spin_lock(&ocs_aes.lock);124 125 if (tctx->aes_dev) {126 aes_dev = tctx->aes_dev;127 goto exit;128 }129 130 /* Only a single OCS device available */131 aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list);132 tctx->aes_dev = aes_dev;133 134exit:135 spin_unlock(&ocs_aes.lock);136 137 return aes_dev;138}139 140/*141 * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual142 * key is being passed in.143 *144 * Return: 0 if key is valid, -EINVAL otherwise.145 */146static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher)147{148 if (!in_key)149 return -EINVAL;150 151 /* For AES, only 128-byte or 256-byte keys are supported. */152 if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 ||153 key_len == OCS_AES_KEYSIZE_256))154 return 0;155 156 /* For SM4, only 128-byte keys are supported. */157 if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128)158 return 0;159 160 /* Everything else is unsupported. */161 return -EINVAL;162}163 164/* Save key into transformation context. */165static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len,166 enum ocs_cipher cipher)167{168 int ret;169 170 ret = check_key(in_key, key_len, cipher);171 if (ret)172 return ret;173 174 memcpy(tctx->key, in_key, key_len);175 tctx->key_len = key_len;176 tctx->cipher = cipher;177 178 return 0;179}180 181/* Set key for symmetric cypher. */182static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key,183 size_t key_len, enum ocs_cipher cipher)184{185 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);186 187 /* Fallback is used for AES with 192-bit key. */188 tctx->use_fallback = (cipher == OCS_AES &&189 key_len == OCS_AES_KEYSIZE_192);190 191 if (!tctx->use_fallback)192 return save_key(tctx, in_key, key_len, cipher);193 194 crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk,195 CRYPTO_TFM_REQ_MASK);196 crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk,197 tfm->base.crt_flags &198 CRYPTO_TFM_REQ_MASK);199 200 return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len);201}202 203/* Set key for AEAD cipher. */204static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,205 size_t key_len, enum ocs_cipher cipher)206{207 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);208 209 /* Fallback is used for AES with 192-bit key. */210 tctx->use_fallback = (cipher == OCS_AES &&211 key_len == OCS_AES_KEYSIZE_192);212 213 if (!tctx->use_fallback)214 return save_key(tctx, in_key, key_len, cipher);215 216 crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);217 crypto_aead_set_flags(tctx->sw_cipher.aead,218 crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);219 220 return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len);221}222 223/* Swap two AES blocks in SG lists. */224static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents,225 off_t blk1_offset, off_t blk2_offset)226{227 u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE];228 229 /*230 * No easy way to copy within sg list, so copy both blocks to temporary231 * buffers first.232 */233 sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset);234 sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset);235 sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset);236 sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset);237}238 239/* Initialize request context to default values. */240static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx)241{242 /* Zero everything. */243 memset(rctx, 0, sizeof(*rctx));244 245 /* Set initial value for DMA addresses. */246 rctx->src_dll.dma_addr = DMA_MAPPING_ERROR;247 rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR;248 rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR;249 rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR;250}251 252static int kmb_ocs_sk_validate_input(struct skcipher_request *req,253 enum ocs_mode mode)254{255 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);256 int iv_size = crypto_skcipher_ivsize(tfm);257 258 switch (mode) {259 case OCS_MODE_ECB:260 /* Ensure input length is multiple of block size */261 if (req->cryptlen % AES_BLOCK_SIZE != 0)262 return -EINVAL;263 264 return 0;265 266 case OCS_MODE_CBC:267 /* Ensure input length is multiple of block size */268 if (req->cryptlen % AES_BLOCK_SIZE != 0)269 return -EINVAL;270 271 /* Ensure IV is present and block size in length */272 if (!req->iv || iv_size != AES_BLOCK_SIZE)273 return -EINVAL;274 /*275 * NOTE: Since req->cryptlen == 0 case was already handled in276 * kmb_ocs_sk_common(), the above two conditions also guarantee277 * that: cryptlen >= iv_size278 */279 return 0;280 281 case OCS_MODE_CTR:282 /* Ensure IV is present and block size in length */283 if (!req->iv || iv_size != AES_BLOCK_SIZE)284 return -EINVAL;285 return 0;286 287 case OCS_MODE_CTS:288 /* Ensure input length >= block size */289 if (req->cryptlen < AES_BLOCK_SIZE)290 return -EINVAL;291 292 /* Ensure IV is present and block size in length */293 if (!req->iv || iv_size != AES_BLOCK_SIZE)294 return -EINVAL;295 296 return 0;297 default:298 return -EINVAL;299 }300}301 302/*303 * Called by encrypt() / decrypt() skcipher functions.304 *305 * Use fallback if needed, otherwise initialize context and enqueue request306 * into engine.307 */308static int kmb_ocs_sk_common(struct skcipher_request *req,309 enum ocs_cipher cipher,310 enum ocs_instruction instruction,311 enum ocs_mode mode)312{313 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);314 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);315 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);316 struct ocs_aes_dev *aes_dev;317 int rc;318 319 if (tctx->use_fallback) {320 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk);321 322 skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk);323 skcipher_request_set_callback(subreq, req->base.flags, NULL,324 NULL);325 skcipher_request_set_crypt(subreq, req->src, req->dst,326 req->cryptlen, req->iv);327 328 if (instruction == OCS_ENCRYPT)329 rc = crypto_skcipher_encrypt(subreq);330 else331 rc = crypto_skcipher_decrypt(subreq);332 333 skcipher_request_zero(subreq);334 335 return rc;336 }337 338 /*339 * If cryptlen == 0, no processing needed for ECB, CBC and CTR.340 *341 * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL.342 */343 if (!req->cryptlen && mode != OCS_MODE_CTS)344 return 0;345 346 rc = kmb_ocs_sk_validate_input(req, mode);347 if (rc)348 return rc;349 350 aes_dev = kmb_ocs_aes_find_dev(tctx);351 if (!aes_dev)352 return -ENODEV;353 354 if (cipher != tctx->cipher)355 return -EINVAL;356 357 ocs_aes_init_rctx(rctx);358 rctx->instruction = instruction;359 rctx->mode = mode;360 361 return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req);362}363 364static void cleanup_ocs_dma_linked_list(struct device *dev,365 struct ocs_dll_desc *dll)366{367 if (dll->vaddr)368 dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr);369 dll->vaddr = NULL;370 dll->size = 0;371 dll->dma_addr = DMA_MAPPING_ERROR;372}373 374static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req)375{376 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);377 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);378 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);379 struct device *dev = tctx->aes_dev->dev;380 381 if (rctx->src_dma_count) {382 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);383 rctx->src_dma_count = 0;384 }385 386 if (rctx->dst_dma_count) {387 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?388 DMA_BIDIRECTIONAL :389 DMA_FROM_DEVICE);390 rctx->dst_dma_count = 0;391 }392 393 /* Clean up OCS DMA linked lists */394 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);395 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);396}397 398static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req)399{400 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);401 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);402 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);403 int iv_size = crypto_skcipher_ivsize(tfm);404 int rc;405 406 /*407 * For CBC decrypt, save last block (iv) to last_ct_blk buffer.408 *409 * Note: if we are here, we already checked that cryptlen >= iv_size410 * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see411 * kmb_ocs_sk_validate_input().412 */413 if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT)414 scatterwalk_map_and_copy(rctx->last_ct_blk, req->src,415 req->cryptlen - iv_size, iv_size, 0);416 417 /* For CTS decrypt, swap last two blocks, if needed. */418 if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT)419 sg_swap_blocks(req->dst, rctx->dst_nents,420 req->cryptlen - AES_BLOCK_SIZE,421 req->cryptlen - (2 * AES_BLOCK_SIZE));422 423 /* src and dst buffers are the same, use bidirectional DMA mapping. */424 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,425 rctx->dst_nents, DMA_BIDIRECTIONAL);426 if (rctx->dst_dma_count == 0) {427 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");428 return -ENOMEM;429 }430 431 /* Create DST linked list */432 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,433 rctx->dst_dma_count, &rctx->dst_dll,434 req->cryptlen, 0);435 if (rc)436 return rc;437 /*438 * If descriptor creation was successful, set the src_dll.dma_addr to439 * the value of dst_dll.dma_addr, as we do in-place AES operation on440 * the src.441 */442 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;443 444 return 0;445}446 447static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req)448{449 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);450 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);451 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);452 int rc;453 454 rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen);455 if (rctx->src_nents < 0)456 return -EBADMSG;457 458 /* Map SRC SG. */459 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,460 rctx->src_nents, DMA_TO_DEVICE);461 if (rctx->src_dma_count == 0) {462 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");463 return -ENOMEM;464 }465 466 /* Create SRC linked list */467 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,468 rctx->src_dma_count, &rctx->src_dll,469 req->cryptlen, 0);470 if (rc)471 return rc;472 473 /* Map DST SG. */474 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,475 rctx->dst_nents, DMA_FROM_DEVICE);476 if (rctx->dst_dma_count == 0) {477 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");478 return -ENOMEM;479 }480 481 /* Create DST linked list */482 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,483 rctx->dst_dma_count, &rctx->dst_dll,484 req->cryptlen, 0);485 if (rc)486 return rc;487 488 /* If this is not a CTS decrypt operation with swapping, we are done. */489 if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT))490 return 0;491 492 /*493 * Otherwise, we have to copy src to dst (as we cannot modify src).494 * Use OCS AES bypass mode to copy src to dst via DMA.495 *496 * NOTE: for anything other than small data sizes this is rather497 * inefficient.498 */499 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr,500 rctx->src_dll.dma_addr, req->cryptlen);501 if (rc)502 return rc;503 504 /*505 * Now dst == src, so clean up what we did so far and use in_place506 * logic.507 */508 kmb_ocs_sk_dma_cleanup(req);509 rctx->in_place = true;510 511 return kmb_ocs_sk_prepare_inplace(req);512}513 514static int kmb_ocs_sk_run(struct skcipher_request *req)515{516 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);517 struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);518 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);519 struct ocs_aes_dev *aes_dev = tctx->aes_dev;520 int iv_size = crypto_skcipher_ivsize(tfm);521 int rc;522 523 rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen);524 if (rctx->dst_nents < 0)525 return -EBADMSG;526 527 /*528 * If 2 blocks or greater, and multiple of block size swap last two529 * blocks to be compatible with other crypto API CTS implementations:530 * OCS mode uses CBC-CS2, whereas other crypto API implementations use531 * CBC-CS3.532 * CBC-CS2 and CBC-CS3 defined by:533 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf534 */535 rctx->cts_swap = (rctx->mode == OCS_MODE_CTS &&536 req->cryptlen > AES_BLOCK_SIZE &&537 req->cryptlen % AES_BLOCK_SIZE == 0);538 539 rctx->in_place = (req->src == req->dst);540 541 if (rctx->in_place)542 rc = kmb_ocs_sk_prepare_inplace(req);543 else544 rc = kmb_ocs_sk_prepare_notinplace(req);545 546 if (rc)547 goto error;548 549 rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction,550 rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr,551 req->cryptlen, req->iv, iv_size);552 if (rc)553 goto error;554 555 /* Clean-up DMA before further processing output. */556 kmb_ocs_sk_dma_cleanup(req);557 558 /* For CTS Encrypt, swap last 2 blocks, if needed. */559 if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) {560 sg_swap_blocks(req->dst, rctx->dst_nents,561 req->cryptlen - AES_BLOCK_SIZE,562 req->cryptlen - (2 * AES_BLOCK_SIZE));563 return 0;564 }565 566 /* For CBC copy IV to req->IV. */567 if (rctx->mode == OCS_MODE_CBC) {568 /* CBC encrypt case. */569 if (rctx->instruction == OCS_ENCRYPT) {570 scatterwalk_map_and_copy(req->iv, req->dst,571 req->cryptlen - iv_size,572 iv_size, 0);573 return 0;574 }575 /* CBC decrypt case. */576 if (rctx->in_place)577 memcpy(req->iv, rctx->last_ct_blk, iv_size);578 else579 scatterwalk_map_and_copy(req->iv, req->src,580 req->cryptlen - iv_size,581 iv_size, 0);582 return 0;583 }584 /* For all other modes there's nothing to do. */585 586 return 0;587 588error:589 kmb_ocs_sk_dma_cleanup(req);590 591 return rc;592}593 594static int kmb_ocs_aead_validate_input(struct aead_request *req,595 enum ocs_instruction instruction,596 enum ocs_mode mode)597{598 struct crypto_aead *tfm = crypto_aead_reqtfm(req);599 int tag_size = crypto_aead_authsize(tfm);600 int iv_size = crypto_aead_ivsize(tfm);601 602 /* For decrypt crytplen == len(PT) + len(tag). */603 if (instruction == OCS_DECRYPT && req->cryptlen < tag_size)604 return -EINVAL;605 606 /* IV is mandatory. */607 if (!req->iv)608 return -EINVAL;609 610 switch (mode) {611 case OCS_MODE_GCM:612 if (iv_size != GCM_AES_IV_SIZE)613 return -EINVAL;614 615 return 0;616 617 case OCS_MODE_CCM:618 /* Ensure IV is present and block size in length */619 if (iv_size != AES_BLOCK_SIZE)620 return -EINVAL;621 622 return 0;623 624 default:625 return -EINVAL;626 }627}628 629/*630 * Called by encrypt() / decrypt() aead functions.631 *632 * Use fallback if needed, otherwise initialize context and enqueue request633 * into engine.634 */635static int kmb_ocs_aead_common(struct aead_request *req,636 enum ocs_cipher cipher,637 enum ocs_instruction instruction,638 enum ocs_mode mode)639{640 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));641 struct ocs_aes_rctx *rctx = aead_request_ctx(req);642 struct ocs_aes_dev *dd;643 int rc;644 645 if (tctx->use_fallback) {646 struct aead_request *subreq = aead_request_ctx(req);647 648 aead_request_set_tfm(subreq, tctx->sw_cipher.aead);649 aead_request_set_callback(subreq, req->base.flags,650 req->base.complete, req->base.data);651 aead_request_set_crypt(subreq, req->src, req->dst,652 req->cryptlen, req->iv);653 aead_request_set_ad(subreq, req->assoclen);654 rc = crypto_aead_setauthsize(tctx->sw_cipher.aead,655 crypto_aead_authsize(crypto_aead_reqtfm(req)));656 if (rc)657 return rc;658 659 return (instruction == OCS_ENCRYPT) ?660 crypto_aead_encrypt(subreq) :661 crypto_aead_decrypt(subreq);662 }663 664 rc = kmb_ocs_aead_validate_input(req, instruction, mode);665 if (rc)666 return rc;667 668 dd = kmb_ocs_aes_find_dev(tctx);669 if (!dd)670 return -ENODEV;671 672 if (cipher != tctx->cipher)673 return -EINVAL;674 675 ocs_aes_init_rctx(rctx);676 rctx->instruction = instruction;677 rctx->mode = mode;678 679 return crypto_transfer_aead_request_to_engine(dd->engine, req);680}681 682static void kmb_ocs_aead_dma_cleanup(struct aead_request *req)683{684 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));685 struct ocs_aes_rctx *rctx = aead_request_ctx(req);686 struct device *dev = tctx->aes_dev->dev;687 688 if (rctx->src_dma_count) {689 dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);690 rctx->src_dma_count = 0;691 }692 693 if (rctx->dst_dma_count) {694 dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?695 DMA_BIDIRECTIONAL :696 DMA_FROM_DEVICE);697 rctx->dst_dma_count = 0;698 }699 /* Clean up OCS DMA linked lists */700 cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);701 cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);702 cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll);703 cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll);704}705 706/**707 * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing.708 * @req: The AEAD request being processed.709 * @src_dll_size: Where to store the length of the data mapped into the710 * src_dll OCS DMA list.711 *712 * Do the following:713 * - DMA map req->src and req->dst714 * - Initialize the following OCS DMA linked lists: rctx->src_dll,715 * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll.716 *717 * Return: 0 on success, negative error code otherwise.718 */719static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size)720{721 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));722 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));723 struct ocs_aes_rctx *rctx = aead_request_ctx(req);724 u32 in_size; /* The length of the data to be mapped by src_dll. */725 u32 out_size; /* The length of the data to be mapped by dst_dll. */726 u32 dst_size; /* The length of the data in dst_sg. */727 int rc;728 729 /* Get number of entries in input data SG list. */730 rctx->src_nents = sg_nents_for_len(req->src,731 req->assoclen + req->cryptlen);732 if (rctx->src_nents < 0)733 return -EBADMSG;734 735 if (rctx->instruction == OCS_DECRYPT) {736 /*737 * For decrypt:738 * - src sg list is: AAD|CT|tag739 * - dst sg list expects: AAD|PT740 *741 * in_size == len(CT); out_size == len(PT)742 */743 744 /* req->cryptlen includes both CT and tag. */745 in_size = req->cryptlen - tag_size;746 747 /* out_size = PT size == CT size */748 out_size = in_size;749 750 /* len(dst_sg) == len(AAD) + len(PT) */751 dst_size = req->assoclen + out_size;752 753 /*754 * Copy tag from source SG list to 'in_tag' buffer.755 *756 * Note: this needs to be done here, before DMA mapping src_sg.757 */758 sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag,759 tag_size, req->assoclen + in_size);760 761 } else { /* OCS_ENCRYPT */762 /*763 * For encrypt:764 * src sg list is: AAD|PT765 * dst sg list expects: AAD|CT|tag766 */767 /* in_size == len(PT) */768 in_size = req->cryptlen;769 770 /*771 * In CCM mode the OCS engine appends the tag to the ciphertext,772 * but in GCM mode the tag must be read from the tag registers773 * and appended manually below774 */775 out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size :776 in_size;777 /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */778 dst_size = req->assoclen + in_size + tag_size;779 }780 *src_dll_size = in_size;781 782 /* Get number of entries in output data SG list. */783 rctx->dst_nents = sg_nents_for_len(req->dst, dst_size);784 if (rctx->dst_nents < 0)785 return -EBADMSG;786 787 rctx->in_place = (req->src == req->dst) ? 1 : 0;788 789 /* Map destination; use bidirectional mapping for in-place case. */790 rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,791 rctx->dst_nents,792 rctx->in_place ? DMA_BIDIRECTIONAL :793 DMA_FROM_DEVICE);794 if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) {795 dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");796 return -ENOMEM;797 }798 799 /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */800 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,801 rctx->dst_dma_count,802 &rctx->aad_dst_dll, req->assoclen,803 0);804 if (rc)805 return rc;806 807 /* Create DST list: maps dst[AAD_SIZE:out_size] */808 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,809 rctx->dst_dma_count, &rctx->dst_dll,810 out_size, req->assoclen);811 if (rc)812 return rc;813 814 if (rctx->in_place) {815 /* If this is not CCM encrypt, we are done. */816 if (!(rctx->mode == OCS_MODE_CCM &&817 rctx->instruction == OCS_ENCRYPT)) {818 /*819 * SRC and DST are the same, so re-use the same DMA820 * addresses (to avoid allocating new DMA lists821 * identical to the dst ones).822 */823 rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;824 rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr;825 826 return 0;827 }828 /*829 * For CCM encrypt the input and output linked lists contain830 * different amounts of data, so, we need to create different831 * SRC and AAD SRC lists, even for the in-place case.832 */833 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,834 rctx->dst_dma_count,835 &rctx->aad_src_dll,836 req->assoclen, 0);837 if (rc)838 return rc;839 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,840 rctx->dst_dma_count,841 &rctx->src_dll, in_size,842 req->assoclen);843 if (rc)844 return rc;845 846 return 0;847 }848 /* Not in-place case. */849 850 /* Map source SG. */851 rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,852 rctx->src_nents, DMA_TO_DEVICE);853 if (rctx->src_dma_count == 0 && rctx->src_nents != 0) {854 dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");855 return -ENOMEM;856 }857 858 /* Create AAD SRC list. */859 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,860 rctx->src_dma_count,861 &rctx->aad_src_dll,862 req->assoclen, 0);863 if (rc)864 return rc;865 866 /* Create SRC list. */867 rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,868 rctx->src_dma_count,869 &rctx->src_dll, in_size,870 req->assoclen);871 if (rc)872 return rc;873 874 if (req->assoclen == 0)875 return 0;876 877 /* Copy AAD from src sg to dst sg using OCS DMA. */878 rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr,879 rctx->aad_src_dll.dma_addr, req->cryptlen);880 if (rc)881 dev_err(tctx->aes_dev->dev,882 "Failed to copy source AAD to destination AAD\n");883 884 return rc;885}886 887static int kmb_ocs_aead_run(struct aead_request *req)888{889 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));890 const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));891 struct ocs_aes_rctx *rctx = aead_request_ctx(req);892 u32 in_size; /* The length of the data mapped by src_dll. */893 int rc;894 895 rc = kmb_ocs_aead_dma_prepare(req, &in_size);896 if (rc)897 goto exit;898 899 /* For CCM, we just call the OCS processing and we are done. */900 if (rctx->mode == OCS_MODE_CCM) {901 rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher,902 rctx->instruction, rctx->dst_dll.dma_addr,903 rctx->src_dll.dma_addr, in_size,904 req->iv,905 rctx->aad_src_dll.dma_addr, req->assoclen,906 rctx->in_tag, tag_size);907 goto exit;908 }909 /* GCM case; invoke OCS processing. */910 rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher,911 rctx->instruction,912 rctx->dst_dll.dma_addr,913 rctx->src_dll.dma_addr, in_size,914 req->iv,915 rctx->aad_src_dll.dma_addr, req->assoclen,916 rctx->out_tag, tag_size);917 if (rc)918 goto exit;919 920 /* For GCM decrypt, we have to compare in_tag with out_tag. */921 if (rctx->instruction == OCS_DECRYPT) {922 rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ?923 -EBADMSG : 0;924 goto exit;925 }926 927 /* For GCM encrypt, we must manually copy out_tag to DST sg. */928 929 /* Clean-up must be called before the sg_pcopy_from_buffer() below. */930 kmb_ocs_aead_dma_cleanup(req);931 932 /* Copy tag to destination sg after AAD and CT. */933 sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag,934 tag_size, req->assoclen + req->cryptlen);935 936 /* Return directly as DMA cleanup already done. */937 return 0;938 939exit:940 kmb_ocs_aead_dma_cleanup(req);941 942 return rc;943}944 945static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine,946 void *areq)947{948 struct skcipher_request *req =949 container_of(areq, struct skcipher_request, base);950 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);951 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);952 int err;953 954 if (!tctx->aes_dev) {955 err = -ENODEV;956 goto exit;957 }958 959 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,960 tctx->cipher);961 if (err)962 goto exit;963 964 err = kmb_ocs_sk_run(req);965 966exit:967 crypto_finalize_skcipher_request(engine, req, err);968 969 return 0;970}971 972static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine,973 void *areq)974{975 struct aead_request *req = container_of(areq,976 struct aead_request, base);977 struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));978 int err;979 980 if (!tctx->aes_dev)981 return -ENODEV;982 983 err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,984 tctx->cipher);985 if (err)986 goto exit;987 988 err = kmb_ocs_aead_run(req);989 990exit:991 crypto_finalize_aead_request(tctx->aes_dev->engine, req, err);992 993 return 0;994}995 996static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,997 unsigned int key_len)998{999 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES);1000}1001 1002static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,1003 unsigned int key_len)1004{1005 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES);1006}1007 1008#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB1009static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req)1010{1011 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB);1012}1013 1014static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req)1015{1016 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB);1017}1018#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */1019 1020static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req)1021{1022 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC);1023}1024 1025static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req)1026{1027 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC);1028}1029 1030static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req)1031{1032 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR);1033}1034 1035static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req)1036{1037 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR);1038}1039 1040#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS1041static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req)1042{1043 return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS);1044}1045 1046static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req)1047{1048 return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS);1049}1050#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */1051 1052static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req)1053{1054 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM);1055}1056 1057static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req)1058{1059 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM);1060}1061 1062static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req)1063{1064 return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM);1065}1066 1067static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req)1068{1069 return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM);1070}1071 1072static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key,1073 unsigned int key_len)1074{1075 return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4);1076}1077 1078static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,1079 unsigned int key_len)1080{1081 return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4);1082}1083 1084#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB1085static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req)1086{1087 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB);1088}1089 1090static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req)1091{1092 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB);1093}1094#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */1095 1096static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req)1097{1098 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC);1099}1100 1101static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req)1102{1103 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC);1104}1105 1106static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req)1107{1108 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR);1109}1110 1111static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req)1112{1113 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR);1114}1115 1116#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS1117static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req)1118{1119 return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS);1120}1121 1122static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req)1123{1124 return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS);1125}1126#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */1127 1128static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req)1129{1130 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM);1131}1132 1133static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req)1134{1135 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM);1136}1137 1138static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req)1139{1140 return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM);1141}1142 1143static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req)1144{1145 return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM);1146}1147 1148static int ocs_aes_init_tfm(struct crypto_skcipher *tfm)1149{1150 const char *alg_name = crypto_tfm_alg_name(&tfm->base);1151 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);1152 struct crypto_sync_skcipher *blk;1153 1154 /* set fallback cipher in case it will be needed */1155 blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);1156 if (IS_ERR(blk))1157 return PTR_ERR(blk);1158 1159 tctx->sw_cipher.sk = blk;1160 1161 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));1162 1163 return 0;1164}1165 1166static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm)1167{1168 crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));1169 1170 return 0;1171}1172 1173static inline void clear_key(struct ocs_aes_tctx *tctx)1174{1175 memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256);1176 1177 /* Zero key registers if set */1178 if (tctx->aes_dev)1179 ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256,1180 tctx->key, OCS_AES);1181}1182 1183static void ocs_exit_tfm(struct crypto_skcipher *tfm)1184{1185 struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);1186 1187 clear_key(tctx);1188 1189 if (tctx->sw_cipher.sk) {1190 crypto_free_sync_skcipher(tctx->sw_cipher.sk);1191 tctx->sw_cipher.sk = NULL;1192 }1193}1194 1195static int ocs_aes_aead_cra_init(struct crypto_aead *tfm)1196{1197 const char *alg_name = crypto_tfm_alg_name(&tfm->base);1198 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);1199 struct crypto_aead *blk;1200 1201 /* Set fallback cipher in case it will be needed */1202 blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);1203 if (IS_ERR(blk))1204 return PTR_ERR(blk);1205 1206 tctx->sw_cipher.aead = blk;1207 1208 crypto_aead_set_reqsize(tfm,1209 max(sizeof(struct ocs_aes_rctx),1210 (sizeof(struct aead_request) +1211 crypto_aead_reqsize(tctx->sw_cipher.aead))));1212 1213 return 0;1214}1215 1216static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm,1217 unsigned int authsize)1218{1219 switch (authsize) {1220 case 4:1221 case 6:1222 case 8:1223 case 10:1224 case 12:1225 case 14:1226 case 16:1227 return 0;1228 default:1229 return -EINVAL;1230 }1231}1232 1233static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm,1234 unsigned int authsize)1235{1236 return crypto_gcm_check_authsize(authsize);1237}1238 1239static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm)1240{1241 crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));1242 1243 return 0;1244}1245 1246static void ocs_aead_cra_exit(struct crypto_aead *tfm)1247{1248 struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);1249 1250 clear_key(tctx);1251 1252 if (tctx->sw_cipher.aead) {1253 crypto_free_aead(tctx->sw_cipher.aead);1254 tctx->sw_cipher.aead = NULL;1255 }1256}1257 1258static struct skcipher_engine_alg algs[] = {1259#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB1260 {1261 .base.base.cra_name = "ecb(aes)",1262 .base.base.cra_driver_name = "ecb-aes-keembay-ocs",1263 .base.base.cra_priority = KMB_OCS_PRIORITY,1264 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1265 CRYPTO_ALG_KERN_DRIVER_ONLY |1266 CRYPTO_ALG_NEED_FALLBACK,1267 .base.base.cra_blocksize = AES_BLOCK_SIZE,1268 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1269 .base.base.cra_module = THIS_MODULE,1270 .base.base.cra_alignmask = 0,1271 1272 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,1273 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,1274 .base.setkey = kmb_ocs_aes_set_key,1275 .base.encrypt = kmb_ocs_aes_ecb_encrypt,1276 .base.decrypt = kmb_ocs_aes_ecb_decrypt,1277 .base.init = ocs_aes_init_tfm,1278 .base.exit = ocs_exit_tfm,1279 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1280 },1281#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */1282 {1283 .base.base.cra_name = "cbc(aes)",1284 .base.base.cra_driver_name = "cbc-aes-keembay-ocs",1285 .base.base.cra_priority = KMB_OCS_PRIORITY,1286 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1287 CRYPTO_ALG_KERN_DRIVER_ONLY |1288 CRYPTO_ALG_NEED_FALLBACK,1289 .base.base.cra_blocksize = AES_BLOCK_SIZE,1290 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1291 .base.base.cra_module = THIS_MODULE,1292 .base.base.cra_alignmask = 0,1293 1294 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,1295 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,1296 .base.ivsize = AES_BLOCK_SIZE,1297 .base.setkey = kmb_ocs_aes_set_key,1298 .base.encrypt = kmb_ocs_aes_cbc_encrypt,1299 .base.decrypt = kmb_ocs_aes_cbc_decrypt,1300 .base.init = ocs_aes_init_tfm,1301 .base.exit = ocs_exit_tfm,1302 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1303 },1304 {1305 .base.base.cra_name = "ctr(aes)",1306 .base.base.cra_driver_name = "ctr-aes-keembay-ocs",1307 .base.base.cra_priority = KMB_OCS_PRIORITY,1308 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1309 CRYPTO_ALG_KERN_DRIVER_ONLY |1310 CRYPTO_ALG_NEED_FALLBACK,1311 .base.base.cra_blocksize = 1,1312 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1313 .base.base.cra_module = THIS_MODULE,1314 .base.base.cra_alignmask = 0,1315 1316 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,1317 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,1318 .base.ivsize = AES_BLOCK_SIZE,1319 .base.setkey = kmb_ocs_aes_set_key,1320 .base.encrypt = kmb_ocs_aes_ctr_encrypt,1321 .base.decrypt = kmb_ocs_aes_ctr_decrypt,1322 .base.init = ocs_aes_init_tfm,1323 .base.exit = ocs_exit_tfm,1324 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1325 },1326#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS1327 {1328 .base.base.cra_name = "cts(cbc(aes))",1329 .base.base.cra_driver_name = "cts-aes-keembay-ocs",1330 .base.base.cra_priority = KMB_OCS_PRIORITY,1331 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1332 CRYPTO_ALG_KERN_DRIVER_ONLY |1333 CRYPTO_ALG_NEED_FALLBACK,1334 .base.base.cra_blocksize = AES_BLOCK_SIZE,1335 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1336 .base.base.cra_module = THIS_MODULE,1337 .base.base.cra_alignmask = 0,1338 1339 .base.min_keysize = OCS_AES_MIN_KEY_SIZE,1340 .base.max_keysize = OCS_AES_MAX_KEY_SIZE,1341 .base.ivsize = AES_BLOCK_SIZE,1342 .base.setkey = kmb_ocs_aes_set_key,1343 .base.encrypt = kmb_ocs_aes_cts_encrypt,1344 .base.decrypt = kmb_ocs_aes_cts_decrypt,1345 .base.init = ocs_aes_init_tfm,1346 .base.exit = ocs_exit_tfm,1347 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1348 },1349#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */1350#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB1351 {1352 .base.base.cra_name = "ecb(sm4)",1353 .base.base.cra_driver_name = "ecb-sm4-keembay-ocs",1354 .base.base.cra_priority = KMB_OCS_PRIORITY,1355 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1356 CRYPTO_ALG_KERN_DRIVER_ONLY,1357 .base.base.cra_blocksize = AES_BLOCK_SIZE,1358 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1359 .base.base.cra_module = THIS_MODULE,1360 .base.base.cra_alignmask = 0,1361 1362 .base.min_keysize = OCS_SM4_KEY_SIZE,1363 .base.max_keysize = OCS_SM4_KEY_SIZE,1364 .base.setkey = kmb_ocs_sm4_set_key,1365 .base.encrypt = kmb_ocs_sm4_ecb_encrypt,1366 .base.decrypt = kmb_ocs_sm4_ecb_decrypt,1367 .base.init = ocs_sm4_init_tfm,1368 .base.exit = ocs_exit_tfm,1369 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1370 },1371#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */1372 {1373 .base.base.cra_name = "cbc(sm4)",1374 .base.base.cra_driver_name = "cbc-sm4-keembay-ocs",1375 .base.base.cra_priority = KMB_OCS_PRIORITY,1376 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1377 CRYPTO_ALG_KERN_DRIVER_ONLY,1378 .base.base.cra_blocksize = AES_BLOCK_SIZE,1379 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1380 .base.base.cra_module = THIS_MODULE,1381 .base.base.cra_alignmask = 0,1382 1383 .base.min_keysize = OCS_SM4_KEY_SIZE,1384 .base.max_keysize = OCS_SM4_KEY_SIZE,1385 .base.ivsize = AES_BLOCK_SIZE,1386 .base.setkey = kmb_ocs_sm4_set_key,1387 .base.encrypt = kmb_ocs_sm4_cbc_encrypt,1388 .base.decrypt = kmb_ocs_sm4_cbc_decrypt,1389 .base.init = ocs_sm4_init_tfm,1390 .base.exit = ocs_exit_tfm,1391 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1392 },1393 {1394 .base.base.cra_name = "ctr(sm4)",1395 .base.base.cra_driver_name = "ctr-sm4-keembay-ocs",1396 .base.base.cra_priority = KMB_OCS_PRIORITY,1397 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1398 CRYPTO_ALG_KERN_DRIVER_ONLY,1399 .base.base.cra_blocksize = 1,1400 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1401 .base.base.cra_module = THIS_MODULE,1402 .base.base.cra_alignmask = 0,1403 1404 .base.min_keysize = OCS_SM4_KEY_SIZE,1405 .base.max_keysize = OCS_SM4_KEY_SIZE,1406 .base.ivsize = AES_BLOCK_SIZE,1407 .base.setkey = kmb_ocs_sm4_set_key,1408 .base.encrypt = kmb_ocs_sm4_ctr_encrypt,1409 .base.decrypt = kmb_ocs_sm4_ctr_decrypt,1410 .base.init = ocs_sm4_init_tfm,1411 .base.exit = ocs_exit_tfm,1412 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1413 },1414#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS1415 {1416 .base.base.cra_name = "cts(cbc(sm4))",1417 .base.base.cra_driver_name = "cts-sm4-keembay-ocs",1418 .base.base.cra_priority = KMB_OCS_PRIORITY,1419 .base.base.cra_flags = CRYPTO_ALG_ASYNC |1420 CRYPTO_ALG_KERN_DRIVER_ONLY,1421 .base.base.cra_blocksize = AES_BLOCK_SIZE,1422 .base.base.cra_ctxsize = sizeof(struct ocs_aes_tctx),1423 .base.base.cra_module = THIS_MODULE,1424 .base.base.cra_alignmask = 0,1425 1426 .base.min_keysize = OCS_SM4_KEY_SIZE,1427 .base.max_keysize = OCS_SM4_KEY_SIZE,1428 .base.ivsize = AES_BLOCK_SIZE,1429 .base.setkey = kmb_ocs_sm4_set_key,1430 .base.encrypt = kmb_ocs_sm4_cts_encrypt,1431 .base.decrypt = kmb_ocs_sm4_cts_decrypt,1432 .base.init = ocs_sm4_init_tfm,1433 .base.exit = ocs_exit_tfm,1434 .op.do_one_request = kmb_ocs_aes_sk_do_one_request,1435 }1436#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */1437};1438 1439static struct aead_engine_alg algs_aead[] = {1440 {1441 .base.base = {1442 .cra_name = "gcm(aes)",1443 .cra_driver_name = "gcm-aes-keembay-ocs",1444 .cra_priority = KMB_OCS_PRIORITY,1445 .cra_flags = CRYPTO_ALG_ASYNC |1446 CRYPTO_ALG_KERN_DRIVER_ONLY |1447 CRYPTO_ALG_NEED_FALLBACK,1448 .cra_blocksize = 1,1449 .cra_ctxsize = sizeof(struct ocs_aes_tctx),1450 .cra_alignmask = 0,1451 .cra_module = THIS_MODULE,1452 },1453 .base.init = ocs_aes_aead_cra_init,1454 .base.exit = ocs_aead_cra_exit,1455 .base.ivsize = GCM_AES_IV_SIZE,1456 .base.maxauthsize = AES_BLOCK_SIZE,1457 .base.setauthsize = kmb_ocs_aead_gcm_setauthsize,1458 .base.setkey = kmb_ocs_aes_aead_set_key,1459 .base.encrypt = kmb_ocs_aes_gcm_encrypt,1460 .base.decrypt = kmb_ocs_aes_gcm_decrypt,1461 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,1462 },1463 {1464 .base.base = {1465 .cra_name = "ccm(aes)",1466 .cra_driver_name = "ccm-aes-keembay-ocs",1467 .cra_priority = KMB_OCS_PRIORITY,1468 .cra_flags = CRYPTO_ALG_ASYNC |1469 CRYPTO_ALG_KERN_DRIVER_ONLY |1470 CRYPTO_ALG_NEED_FALLBACK,1471 .cra_blocksize = 1,1472 .cra_ctxsize = sizeof(struct ocs_aes_tctx),1473 .cra_alignmask = 0,1474 .cra_module = THIS_MODULE,1475 },1476 .base.init = ocs_aes_aead_cra_init,1477 .base.exit = ocs_aead_cra_exit,1478 .base.ivsize = AES_BLOCK_SIZE,1479 .base.maxauthsize = AES_BLOCK_SIZE,1480 .base.setauthsize = kmb_ocs_aead_ccm_setauthsize,1481 .base.setkey = kmb_ocs_aes_aead_set_key,1482 .base.encrypt = kmb_ocs_aes_ccm_encrypt,1483 .base.decrypt = kmb_ocs_aes_ccm_decrypt,1484 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,1485 },1486 {1487 .base.base = {1488 .cra_name = "gcm(sm4)",1489 .cra_driver_name = "gcm-sm4-keembay-ocs",1490 .cra_priority = KMB_OCS_PRIORITY,1491 .cra_flags = CRYPTO_ALG_ASYNC |1492 CRYPTO_ALG_KERN_DRIVER_ONLY,1493 .cra_blocksize = 1,1494 .cra_ctxsize = sizeof(struct ocs_aes_tctx),1495 .cra_alignmask = 0,1496 .cra_module = THIS_MODULE,1497 },1498 .base.init = ocs_sm4_aead_cra_init,1499 .base.exit = ocs_aead_cra_exit,1500 .base.ivsize = GCM_AES_IV_SIZE,1501 .base.maxauthsize = AES_BLOCK_SIZE,1502 .base.setauthsize = kmb_ocs_aead_gcm_setauthsize,1503 .base.setkey = kmb_ocs_sm4_aead_set_key,1504 .base.encrypt = kmb_ocs_sm4_gcm_encrypt,1505 .base.decrypt = kmb_ocs_sm4_gcm_decrypt,1506 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,1507 },1508 {1509 .base.base = {1510 .cra_name = "ccm(sm4)",1511 .cra_driver_name = "ccm-sm4-keembay-ocs",1512 .cra_priority = KMB_OCS_PRIORITY,1513 .cra_flags = CRYPTO_ALG_ASYNC |1514 CRYPTO_ALG_KERN_DRIVER_ONLY,1515 .cra_blocksize = 1,1516 .cra_ctxsize = sizeof(struct ocs_aes_tctx),1517 .cra_alignmask = 0,1518 .cra_module = THIS_MODULE,1519 },1520 .base.init = ocs_sm4_aead_cra_init,1521 .base.exit = ocs_aead_cra_exit,1522 .base.ivsize = AES_BLOCK_SIZE,1523 .base.maxauthsize = AES_BLOCK_SIZE,1524 .base.setauthsize = kmb_ocs_aead_ccm_setauthsize,1525 .base.setkey = kmb_ocs_sm4_aead_set_key,1526 .base.encrypt = kmb_ocs_sm4_ccm_encrypt,1527 .base.decrypt = kmb_ocs_sm4_ccm_decrypt,1528 .op.do_one_request = kmb_ocs_aes_aead_do_one_request,1529 }1530};1531 1532static void unregister_aes_algs(struct ocs_aes_dev *aes_dev)1533{1534 crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead));1535 crypto_engine_unregister_skciphers(algs, ARRAY_SIZE(algs));1536}1537 1538static int register_aes_algs(struct ocs_aes_dev *aes_dev)1539{1540 int ret;1541 1542 /*1543 * If any algorithm fails to register, all preceding algorithms that1544 * were successfully registered will be automatically unregistered.1545 */1546 ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead));1547 if (ret)1548 return ret;1549 1550 ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs));1551 if (ret)1552 crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs));1553 1554 return ret;1555}1556 1557/* Device tree driver match. */1558static const struct of_device_id kmb_ocs_aes_of_match[] = {1559 {1560 .compatible = "intel,keembay-ocs-aes",1561 },1562 {}1563};1564 1565static void kmb_ocs_aes_remove(struct platform_device *pdev)1566{1567 struct ocs_aes_dev *aes_dev;1568 1569 aes_dev = platform_get_drvdata(pdev);1570 1571 unregister_aes_algs(aes_dev);1572 1573 spin_lock(&ocs_aes.lock);1574 list_del(&aes_dev->list);1575 spin_unlock(&ocs_aes.lock);1576 1577 crypto_engine_exit(aes_dev->engine);1578}1579 1580static int kmb_ocs_aes_probe(struct platform_device *pdev)1581{1582 struct device *dev = &pdev->dev;1583 struct ocs_aes_dev *aes_dev;1584 int rc;1585 1586 aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL);1587 if (!aes_dev)1588 return -ENOMEM;1589 1590 aes_dev->dev = dev;1591 1592 platform_set_drvdata(pdev, aes_dev);1593 1594 rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));1595 if (rc) {1596 dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc);1597 return rc;1598 }1599 1600 /* Get base register address. */1601 aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);1602 if (IS_ERR(aes_dev->base_reg))1603 return PTR_ERR(aes_dev->base_reg);1604 1605 /* Get and request IRQ */1606 aes_dev->irq = platform_get_irq(pdev, 0);1607 if (aes_dev->irq < 0)1608 return aes_dev->irq;1609 1610 rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler,1611 NULL, 0, "keembay-ocs-aes", aes_dev);1612 if (rc < 0) {1613 dev_err(dev, "Could not request IRQ\n");1614 return rc;1615 }1616 1617 INIT_LIST_HEAD(&aes_dev->list);1618 spin_lock(&ocs_aes.lock);1619 list_add_tail(&aes_dev->list, &ocs_aes.dev_list);1620 spin_unlock(&ocs_aes.lock);1621 1622 init_completion(&aes_dev->irq_completion);1623 1624 /* Initialize crypto engine */1625 aes_dev->engine = crypto_engine_alloc_init(dev, true);1626 if (!aes_dev->engine) {1627 rc = -ENOMEM;1628 goto list_del;1629 }1630 1631 rc = crypto_engine_start(aes_dev->engine);1632 if (rc) {1633 dev_err(dev, "Could not start crypto engine\n");1634 goto cleanup;1635 }1636 1637 rc = register_aes_algs(aes_dev);1638 if (rc) {1639 dev_err(dev,1640 "Could not register OCS algorithms with Crypto API\n");1641 goto cleanup;1642 }1643 1644 return 0;1645 1646cleanup:1647 crypto_engine_exit(aes_dev->engine);1648list_del:1649 spin_lock(&ocs_aes.lock);1650 list_del(&aes_dev->list);1651 spin_unlock(&ocs_aes.lock);1652 1653 return rc;1654}1655 1656/* The OCS driver is a platform device. */1657static struct platform_driver kmb_ocs_aes_driver = {1658 .probe = kmb_ocs_aes_probe,1659 .remove_new = kmb_ocs_aes_remove,1660 .driver = {1661 .name = DRV_NAME,1662 .of_match_table = kmb_ocs_aes_of_match,1663 },1664};1665 1666module_platform_driver(kmb_ocs_aes_driver);1667 1668MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver");1669MODULE_LICENSE("GPL");1670 1671MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs");1672MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs");1673MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs");1674MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs");1675 1676MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs");1677MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs");1678MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs");1679MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs");1680 1681#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB1682MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs");1683MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs");1684#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */1685 1686#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS1687MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs");1688MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs");1689#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */1690