451 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Xilinx ZynqMP AES Driver.4 * Copyright (c) 2020 Xilinx Inc.5 */6 7#include <crypto/aes.h>8#include <crypto/engine.h>9#include <crypto/gcm.h>10#include <crypto/internal/aead.h>11#include <crypto/scatterwalk.h>12#include <linux/dma-mapping.h>13#include <linux/err.h>14#include <linux/firmware/xlnx-zynqmp.h>15#include <linux/kernel.h>16#include <linux/module.h>17#include <linux/mod_devicetable.h>18#include <linux/platform_device.h>19#include <linux/string.h>20 21#define ZYNQMP_DMA_BIT_MASK 32U22 23#define ZYNQMP_AES_KEY_SIZE AES_KEYSIZE_25624#define ZYNQMP_AES_AUTH_SIZE 16U25#define ZYNQMP_KEY_SRC_SEL_KEY_LEN 1U26#define ZYNQMP_AES_BLK_SIZE 1U27#define ZYNQMP_AES_MIN_INPUT_BLK_SIZE 4U28#define ZYNQMP_AES_WORD_LEN 4U29 30#define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x0131#define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x1332#define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE30033 34enum zynqmp_aead_op {35 ZYNQMP_AES_DECRYPT = 0,36 ZYNQMP_AES_ENCRYPT37};38 39enum zynqmp_aead_keysrc {40 ZYNQMP_AES_KUP_KEY = 0,41 ZYNQMP_AES_DEV_KEY,42 ZYNQMP_AES_PUF_KEY43};44 45struct zynqmp_aead_drv_ctx {46 union {47 struct aead_engine_alg aead;48 } alg;49 struct device *dev;50 struct crypto_engine *engine;51};52 53struct zynqmp_aead_hw_req {54 u64 src;55 u64 iv;56 u64 key;57 u64 dst;58 u64 size;59 u64 op;60 u64 keysrc;61};62 63struct zynqmp_aead_tfm_ctx {64 struct device *dev;65 u8 key[ZYNQMP_AES_KEY_SIZE];66 u8 *iv;67 u32 keylen;68 u32 authsize;69 enum zynqmp_aead_keysrc keysrc;70 struct crypto_aead *fbk_cipher;71};72 73struct zynqmp_aead_req_ctx {74 enum zynqmp_aead_op op;75};76 77static int zynqmp_aes_aead_cipher(struct aead_request *req)78{79 struct crypto_aead *aead = crypto_aead_reqtfm(req);80 struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);81 struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req);82 struct device *dev = tfm_ctx->dev;83 struct zynqmp_aead_hw_req *hwreq;84 dma_addr_t dma_addr_data, dma_addr_hw_req;85 unsigned int data_size;86 unsigned int status;87 int ret;88 size_t dma_size;89 char *kbuf;90 int err;91 92 if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY)93 dma_size = req->cryptlen + ZYNQMP_AES_KEY_SIZE94 + GCM_AES_IV_SIZE;95 else96 dma_size = req->cryptlen + GCM_AES_IV_SIZE;97 98 kbuf = dma_alloc_coherent(dev, dma_size, &dma_addr_data, GFP_KERNEL);99 if (!kbuf)100 return -ENOMEM;101 102 hwreq = dma_alloc_coherent(dev, sizeof(struct zynqmp_aead_hw_req),103 &dma_addr_hw_req, GFP_KERNEL);104 if (!hwreq) {105 dma_free_coherent(dev, dma_size, kbuf, dma_addr_data);106 return -ENOMEM;107 }108 109 data_size = req->cryptlen;110 scatterwalk_map_and_copy(kbuf, req->src, 0, req->cryptlen, 0);111 memcpy(kbuf + data_size, req->iv, GCM_AES_IV_SIZE);112 113 hwreq->src = dma_addr_data;114 hwreq->dst = dma_addr_data;115 hwreq->iv = hwreq->src + data_size;116 hwreq->keysrc = tfm_ctx->keysrc;117 hwreq->op = rq_ctx->op;118 119 if (hwreq->op == ZYNQMP_AES_ENCRYPT)120 hwreq->size = data_size;121 else122 hwreq->size = data_size - ZYNQMP_AES_AUTH_SIZE;123 124 if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY) {125 memcpy(kbuf + data_size + GCM_AES_IV_SIZE,126 tfm_ctx->key, ZYNQMP_AES_KEY_SIZE);127 128 hwreq->key = hwreq->src + data_size + GCM_AES_IV_SIZE;129 } else {130 hwreq->key = 0;131 }132 133 ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status);134 135 if (ret) {136 dev_err(dev, "ERROR: AES PM API failed\n");137 err = ret;138 } else if (status) {139 switch (status) {140 case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR:141 dev_err(dev, "ERROR: Gcm Tag mismatch\n");142 break;143 case ZYNQMP_AES_WRONG_KEY_SRC_ERR:144 dev_err(dev, "ERROR: Wrong KeySrc, enable secure mode\n");145 break;146 case ZYNQMP_AES_PUF_NOT_PROGRAMMED:147 dev_err(dev, "ERROR: PUF is not registered\n");148 break;149 default:150 dev_err(dev, "ERROR: Unknown error\n");151 break;152 }153 err = -status;154 } else {155 if (hwreq->op == ZYNQMP_AES_ENCRYPT)156 data_size = data_size + ZYNQMP_AES_AUTH_SIZE;157 else158 data_size = data_size - ZYNQMP_AES_AUTH_SIZE;159 160 sg_copy_from_buffer(req->dst, sg_nents(req->dst),161 kbuf, data_size);162 err = 0;163 }164 165 if (kbuf) {166 memzero_explicit(kbuf, dma_size);167 dma_free_coherent(dev, dma_size, kbuf, dma_addr_data);168 }169 if (hwreq) {170 memzero_explicit(hwreq, sizeof(struct zynqmp_aead_hw_req));171 dma_free_coherent(dev, sizeof(struct zynqmp_aead_hw_req),172 hwreq, dma_addr_hw_req);173 }174 return err;175}176 177static int zynqmp_fallback_check(struct zynqmp_aead_tfm_ctx *tfm_ctx,178 struct aead_request *req)179{180 int need_fallback = 0;181 struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req);182 183 if (tfm_ctx->authsize != ZYNQMP_AES_AUTH_SIZE)184 need_fallback = 1;185 186 if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY &&187 tfm_ctx->keylen != ZYNQMP_AES_KEY_SIZE) {188 need_fallback = 1;189 }190 if (req->assoclen != 0 ||191 req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE) {192 need_fallback = 1;193 }194 if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0)195 need_fallback = 1;196 197 if (rq_ctx->op == ZYNQMP_AES_DECRYPT &&198 req->cryptlen <= ZYNQMP_AES_AUTH_SIZE) {199 need_fallback = 1;200 }201 return need_fallback;202}203 204static int zynqmp_handle_aes_req(struct crypto_engine *engine,205 void *req)206{207 struct aead_request *areq =208 container_of(req, struct aead_request, base);209 struct crypto_aead *aead = crypto_aead_reqtfm(req);210 struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);211 struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(areq);212 struct aead_request *subreq = aead_request_ctx(req);213 int need_fallback;214 int err;215 216 need_fallback = zynqmp_fallback_check(tfm_ctx, areq);217 218 if (need_fallback) {219 aead_request_set_tfm(subreq, tfm_ctx->fbk_cipher);220 221 aead_request_set_callback(subreq, areq->base.flags,222 NULL, NULL);223 aead_request_set_crypt(subreq, areq->src, areq->dst,224 areq->cryptlen, areq->iv);225 aead_request_set_ad(subreq, areq->assoclen);226 if (rq_ctx->op == ZYNQMP_AES_ENCRYPT)227 err = crypto_aead_encrypt(subreq);228 else229 err = crypto_aead_decrypt(subreq);230 } else {231 err = zynqmp_aes_aead_cipher(areq);232 }233 234 local_bh_disable();235 crypto_finalize_aead_request(engine, areq, err);236 local_bh_enable();237 238 return 0;239}240 241static int zynqmp_aes_aead_setkey(struct crypto_aead *aead, const u8 *key,242 unsigned int keylen)243{244 struct crypto_tfm *tfm = crypto_aead_tfm(aead);245 struct zynqmp_aead_tfm_ctx *tfm_ctx =246 (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm);247 unsigned char keysrc;248 249 if (keylen == ZYNQMP_KEY_SRC_SEL_KEY_LEN) {250 keysrc = *key;251 if (keysrc == ZYNQMP_AES_KUP_KEY ||252 keysrc == ZYNQMP_AES_DEV_KEY ||253 keysrc == ZYNQMP_AES_PUF_KEY) {254 tfm_ctx->keysrc = (enum zynqmp_aead_keysrc)keysrc;255 } else {256 tfm_ctx->keylen = keylen;257 }258 } else {259 tfm_ctx->keylen = keylen;260 if (keylen == ZYNQMP_AES_KEY_SIZE) {261 tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY;262 memcpy(tfm_ctx->key, key, keylen);263 }264 }265 266 tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;267 tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags &268 CRYPTO_TFM_REQ_MASK);269 270 return crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen);271}272 273static int zynqmp_aes_aead_setauthsize(struct crypto_aead *aead,274 unsigned int authsize)275{276 struct crypto_tfm *tfm = crypto_aead_tfm(aead);277 struct zynqmp_aead_tfm_ctx *tfm_ctx =278 (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm);279 280 tfm_ctx->authsize = authsize;281 return crypto_aead_setauthsize(tfm_ctx->fbk_cipher, authsize);282}283 284static int zynqmp_aes_aead_encrypt(struct aead_request *req)285{286 struct zynqmp_aead_drv_ctx *drv_ctx;287 struct crypto_aead *aead = crypto_aead_reqtfm(req);288 struct aead_alg *alg = crypto_aead_alg(aead);289 struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req);290 291 rq_ctx->op = ZYNQMP_AES_ENCRYPT;292 drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base);293 294 return crypto_transfer_aead_request_to_engine(drv_ctx->engine, req);295}296 297static int zynqmp_aes_aead_decrypt(struct aead_request *req)298{299 struct zynqmp_aead_drv_ctx *drv_ctx;300 struct crypto_aead *aead = crypto_aead_reqtfm(req);301 struct aead_alg *alg = crypto_aead_alg(aead);302 struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req);303 304 rq_ctx->op = ZYNQMP_AES_DECRYPT;305 drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base);306 307 return crypto_transfer_aead_request_to_engine(drv_ctx->engine, req);308}309 310static int zynqmp_aes_aead_init(struct crypto_aead *aead)311{312 struct crypto_tfm *tfm = crypto_aead_tfm(aead);313 struct zynqmp_aead_tfm_ctx *tfm_ctx =314 (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm);315 struct zynqmp_aead_drv_ctx *drv_ctx;316 struct aead_alg *alg = crypto_aead_alg(aead);317 318 drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base);319 tfm_ctx->dev = drv_ctx->dev;320 321 tfm_ctx->fbk_cipher = crypto_alloc_aead(drv_ctx->alg.aead.base.base.cra_name,322 0,323 CRYPTO_ALG_NEED_FALLBACK);324 325 if (IS_ERR(tfm_ctx->fbk_cipher)) {326 pr_err("%s() Error: failed to allocate fallback for %s\n",327 __func__, drv_ctx->alg.aead.base.base.cra_name);328 return PTR_ERR(tfm_ctx->fbk_cipher);329 }330 331 crypto_aead_set_reqsize(aead,332 max(sizeof(struct zynqmp_aead_req_ctx),333 sizeof(struct aead_request) +334 crypto_aead_reqsize(tfm_ctx->fbk_cipher)));335 return 0;336}337 338static void zynqmp_aes_aead_exit(struct crypto_aead *aead)339{340 struct crypto_tfm *tfm = crypto_aead_tfm(aead);341 struct zynqmp_aead_tfm_ctx *tfm_ctx =342 (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm);343 344 if (tfm_ctx->fbk_cipher) {345 crypto_free_aead(tfm_ctx->fbk_cipher);346 tfm_ctx->fbk_cipher = NULL;347 }348 memzero_explicit(tfm_ctx, sizeof(struct zynqmp_aead_tfm_ctx));349}350 351static struct zynqmp_aead_drv_ctx aes_drv_ctx = {352 .alg.aead.base = {353 .setkey = zynqmp_aes_aead_setkey,354 .setauthsize = zynqmp_aes_aead_setauthsize,355 .encrypt = zynqmp_aes_aead_encrypt,356 .decrypt = zynqmp_aes_aead_decrypt,357 .init = zynqmp_aes_aead_init,358 .exit = zynqmp_aes_aead_exit,359 .ivsize = GCM_AES_IV_SIZE,360 .maxauthsize = ZYNQMP_AES_AUTH_SIZE,361 .base = {362 .cra_name = "gcm(aes)",363 .cra_driver_name = "xilinx-zynqmp-aes-gcm",364 .cra_priority = 200,365 .cra_flags = CRYPTO_ALG_TYPE_AEAD |366 CRYPTO_ALG_ASYNC |367 CRYPTO_ALG_ALLOCATES_MEMORY |368 CRYPTO_ALG_KERN_DRIVER_ONLY |369 CRYPTO_ALG_NEED_FALLBACK,370 .cra_blocksize = ZYNQMP_AES_BLK_SIZE,371 .cra_ctxsize = sizeof(struct zynqmp_aead_tfm_ctx),372 .cra_module = THIS_MODULE,373 }374 },375 .alg.aead.op = {376 .do_one_request = zynqmp_handle_aes_req,377 },378};379 380static int zynqmp_aes_aead_probe(struct platform_device *pdev)381{382 struct device *dev = &pdev->dev;383 int err;384 385 /* ZynqMP AES driver supports only one instance */386 if (!aes_drv_ctx.dev)387 aes_drv_ctx.dev = dev;388 else389 return -ENODEV;390 391 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(ZYNQMP_DMA_BIT_MASK));392 if (err < 0) {393 dev_err(dev, "No usable DMA configuration\n");394 return err;395 }396 397 aes_drv_ctx.engine = crypto_engine_alloc_init(dev, 1);398 if (!aes_drv_ctx.engine) {399 dev_err(dev, "Cannot alloc AES engine\n");400 err = -ENOMEM;401 goto err_engine;402 }403 404 err = crypto_engine_start(aes_drv_ctx.engine);405 if (err) {406 dev_err(dev, "Cannot start AES engine\n");407 goto err_engine;408 }409 410 err = crypto_engine_register_aead(&aes_drv_ctx.alg.aead);411 if (err < 0) {412 dev_err(dev, "Failed to register AEAD alg.\n");413 goto err_aead;414 }415 return 0;416 417err_aead:418 crypto_engine_unregister_aead(&aes_drv_ctx.alg.aead);419 420err_engine:421 if (aes_drv_ctx.engine)422 crypto_engine_exit(aes_drv_ctx.engine);423 424 return err;425}426 427static void zynqmp_aes_aead_remove(struct platform_device *pdev)428{429 crypto_engine_exit(aes_drv_ctx.engine);430 crypto_engine_unregister_aead(&aes_drv_ctx.alg.aead);431}432 433static const struct of_device_id zynqmp_aes_dt_ids[] = {434 { .compatible = "xlnx,zynqmp-aes" },435 { /* sentinel */ }436};437MODULE_DEVICE_TABLE(of, zynqmp_aes_dt_ids);438 439static struct platform_driver zynqmp_aes_driver = {440 .probe = zynqmp_aes_aead_probe,441 .remove_new = zynqmp_aes_aead_remove,442 .driver = {443 .name = "zynqmp-aes",444 .of_match_table = zynqmp_aes_dt_ids,445 },446};447 448module_platform_driver(zynqmp_aes_driver);449MODULE_DESCRIPTION("Xilinx ZynqMP AES Driver");450MODULE_LICENSE("GPL");451