511 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * sun8i-ce-cipher.c - hardware cryptographic offloader for4 * Allwinner H3/A64/H5/H2+/H6/R40 SoC5 *6 * Copyright (C) 2016-2019 Corentin LABBE <clabbe.montjoie@gmail.com>7 *8 * This file add support for AES cipher with 128,192,256 bits keysize in9 * CBC and ECB mode.10 *11 * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst12 */13 14#include <linux/bottom_half.h>15#include <linux/crypto.h>16#include <linux/dma-mapping.h>17#include <linux/io.h>18#include <linux/pm_runtime.h>19#include <crypto/scatterwalk.h>20#include <crypto/internal/des.h>21#include <crypto/internal/skcipher.h>22#include "sun8i-ce.h"23 24static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq)25{26 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);27 struct scatterlist *sg;28 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);29 struct sun8i_ce_alg_template *algt;30 unsigned int todo, len;31 32 algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);33 34 if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG ||35 sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) {36 algt->stat_fb_maxsg++;37 return true;38 }39 40 if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) {41 algt->stat_fb_leniv++;42 return true;43 }44 45 if (areq->cryptlen == 0) {46 algt->stat_fb_len0++;47 return true;48 }49 50 if (areq->cryptlen % 16) {51 algt->stat_fb_mod16++;52 return true;53 }54 55 len = areq->cryptlen;56 sg = areq->src;57 while (sg) {58 if (!IS_ALIGNED(sg->offset, sizeof(u32))) {59 algt->stat_fb_srcali++;60 return true;61 }62 todo = min(len, sg->length);63 if (todo % 4) {64 algt->stat_fb_srclen++;65 return true;66 }67 len -= todo;68 sg = sg_next(sg);69 }70 71 len = areq->cryptlen;72 sg = areq->dst;73 while (sg) {74 if (!IS_ALIGNED(sg->offset, sizeof(u32))) {75 algt->stat_fb_dstali++;76 return true;77 }78 todo = min(len, sg->length);79 if (todo % 4) {80 algt->stat_fb_dstlen++;81 return true;82 }83 len -= todo;84 sg = sg_next(sg);85 }86 return false;87}88 89static int sun8i_ce_cipher_fallback(struct skcipher_request *areq)90{91 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);92 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);93 struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);94 int err;95 96 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {97 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);98 struct sun8i_ce_alg_template *algt __maybe_unused;99 100 algt = container_of(alg, struct sun8i_ce_alg_template,101 alg.skcipher.base);102 103#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG104 algt->stat_fb++;105#endif106 }107 108 skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);109 skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,110 areq->base.complete, areq->base.data);111 skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,112 areq->cryptlen, areq->iv);113 if (rctx->op_dir & CE_DECRYPTION)114 err = crypto_skcipher_decrypt(&rctx->fallback_req);115 else116 err = crypto_skcipher_encrypt(&rctx->fallback_req);117 return err;118}119 120static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req)121{122 struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);123 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);124 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);125 struct sun8i_ce_dev *ce = op->ce;126 struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);127 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);128 struct sun8i_ce_alg_template *algt;129 struct sun8i_ce_flow *chan;130 struct ce_task *cet;131 struct scatterlist *sg;132 unsigned int todo, len, offset, ivsize;133 u32 common, sym;134 int flow, i;135 int nr_sgs = 0;136 int nr_sgd = 0;137 int err = 0;138 int ns = sg_nents_for_len(areq->src, areq->cryptlen);139 int nd = sg_nents_for_len(areq->dst, areq->cryptlen);140 141 algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);142 143 dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,144 crypto_tfm_alg_name(areq->base.tfm),145 areq->cryptlen,146 rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),147 op->keylen);148 149#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG150 algt->stat_req++;151#endif152 153 flow = rctx->flow;154 155 chan = &ce->chanlist[flow];156 157 cet = chan->tl;158 memset(cet, 0, sizeof(struct ce_task));159 160 cet->t_id = cpu_to_le32(flow);161 common = ce->variant->alg_cipher[algt->ce_algo_id];162 common |= rctx->op_dir | CE_COMM_INT;163 cet->t_common_ctl = cpu_to_le32(common);164 /* CTS and recent CE (H6) need length in bytes, in word otherwise */165 if (ce->variant->cipher_t_dlen_in_bytes)166 cet->t_dlen = cpu_to_le32(areq->cryptlen);167 else168 cet->t_dlen = cpu_to_le32(areq->cryptlen / 4);169 170 sym = ce->variant->op_mode[algt->ce_blockmode];171 len = op->keylen;172 switch (len) {173 case 128 / 8:174 sym |= CE_AES_128BITS;175 break;176 case 192 / 8:177 sym |= CE_AES_192BITS;178 break;179 case 256 / 8:180 sym |= CE_AES_256BITS;181 break;182 }183 184 cet->t_sym_ctl = cpu_to_le32(sym);185 cet->t_asym_ctl = 0;186 187 rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE);188 if (dma_mapping_error(ce->dev, rctx->addr_key)) {189 dev_err(ce->dev, "Cannot DMA MAP KEY\n");190 err = -EFAULT;191 goto theend;192 }193 cet->t_key = desc_addr_val_le32(ce, rctx->addr_key);194 195 ivsize = crypto_skcipher_ivsize(tfm);196 if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {197 rctx->ivlen = ivsize;198 if (rctx->op_dir & CE_DECRYPTION) {199 offset = areq->cryptlen - ivsize;200 scatterwalk_map_and_copy(chan->backup_iv, areq->src,201 offset, ivsize, 0);202 }203 memcpy(chan->bounce_iv, areq->iv, ivsize);204 rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, rctx->ivlen,205 DMA_TO_DEVICE);206 if (dma_mapping_error(ce->dev, rctx->addr_iv)) {207 dev_err(ce->dev, "Cannot DMA MAP IV\n");208 err = -ENOMEM;209 goto theend_iv;210 }211 cet->t_iv = desc_addr_val_le32(ce, rctx->addr_iv);212 }213 214 if (areq->src == areq->dst) {215 nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL);216 if (nr_sgs <= 0 || nr_sgs > MAX_SG) {217 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);218 err = -EINVAL;219 goto theend_iv;220 }221 nr_sgd = nr_sgs;222 } else {223 nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);224 if (nr_sgs <= 0 || nr_sgs > MAX_SG) {225 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);226 err = -EINVAL;227 goto theend_iv;228 }229 nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE);230 if (nr_sgd <= 0 || nr_sgd > MAX_SG) {231 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd);232 err = -EINVAL;233 goto theend_sgs;234 }235 }236 237 len = areq->cryptlen;238 for_each_sg(areq->src, sg, nr_sgs, i) {239 cet->t_src[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg));240 todo = min(len, sg_dma_len(sg));241 cet->t_src[i].len = cpu_to_le32(todo / 4);242 dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__,243 areq->cryptlen, i, cet->t_src[i].len, sg->offset, todo);244 len -= todo;245 }246 if (len > 0) {247 dev_err(ce->dev, "remaining len %d\n", len);248 err = -EINVAL;249 goto theend_sgs;250 }251 252 len = areq->cryptlen;253 for_each_sg(areq->dst, sg, nr_sgd, i) {254 cet->t_dst[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg));255 todo = min(len, sg_dma_len(sg));256 cet->t_dst[i].len = cpu_to_le32(todo / 4);257 dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__,258 areq->cryptlen, i, cet->t_dst[i].len, sg->offset, todo);259 len -= todo;260 }261 if (len > 0) {262 dev_err(ce->dev, "remaining len %d\n", len);263 err = -EINVAL;264 goto theend_sgs;265 }266 267 chan->timeout = areq->cryptlen;268 rctx->nr_sgs = nr_sgs;269 rctx->nr_sgd = nr_sgd;270 return 0;271 272theend_sgs:273 if (areq->src == areq->dst) {274 dma_unmap_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL);275 } else {276 if (nr_sgs > 0)277 dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);278 dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE);279 }280 281theend_iv:282 if (areq->iv && ivsize > 0) {283 if (rctx->addr_iv)284 dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);285 offset = areq->cryptlen - ivsize;286 if (rctx->op_dir & CE_DECRYPTION) {287 memcpy(areq->iv, chan->backup_iv, ivsize);288 memzero_explicit(chan->backup_iv, ivsize);289 } else {290 scatterwalk_map_and_copy(areq->iv, areq->dst, offset,291 ivsize, 0);292 }293 memzero_explicit(chan->bounce_iv, ivsize);294 }295 296 dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);297 298theend:299 return err;300}301 302static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine,303 void *async_req)304{305 struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);306 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);307 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);308 struct sun8i_ce_dev *ce = op->ce;309 struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);310 struct sun8i_ce_flow *chan;311 struct ce_task *cet;312 unsigned int ivsize, offset;313 int nr_sgs = rctx->nr_sgs;314 int nr_sgd = rctx->nr_sgd;315 int flow;316 317 flow = rctx->flow;318 chan = &ce->chanlist[flow];319 cet = chan->tl;320 ivsize = crypto_skcipher_ivsize(tfm);321 322 if (areq->src == areq->dst) {323 dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL);324 } else {325 if (nr_sgs > 0)326 dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE);327 dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE);328 }329 330 if (areq->iv && ivsize > 0) {331 if (cet->t_iv)332 dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);333 offset = areq->cryptlen - ivsize;334 if (rctx->op_dir & CE_DECRYPTION) {335 memcpy(areq->iv, chan->backup_iv, ivsize);336 memzero_explicit(chan->backup_iv, ivsize);337 } else {338 scatterwalk_map_and_copy(areq->iv, areq->dst, offset,339 ivsize, 0);340 }341 memzero_explicit(chan->bounce_iv, ivsize);342 }343 344 dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);345}346 347static void sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq)348{349 struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);350 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq);351 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);352 struct sun8i_ce_dev *ce = op->ce;353 struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq);354 int flow, err;355 356 flow = rctx->flow;357 err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm));358 sun8i_ce_cipher_unprepare(engine, areq);359 local_bh_disable();360 crypto_finalize_skcipher_request(engine, breq, err);361 local_bh_enable();362}363 364int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq)365{366 int err = sun8i_ce_cipher_prepare(engine, areq);367 368 if (err)369 return err;370 371 sun8i_ce_cipher_run(engine, areq);372 return 0;373}374 375int sun8i_ce_skdecrypt(struct skcipher_request *areq)376{377 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);378 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);379 struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);380 struct crypto_engine *engine;381 int e;382 383 rctx->op_dir = CE_DECRYPTION;384 if (sun8i_ce_cipher_need_fallback(areq))385 return sun8i_ce_cipher_fallback(areq);386 387 e = sun8i_ce_get_engine_number(op->ce);388 rctx->flow = e;389 engine = op->ce->chanlist[e].engine;390 391 return crypto_transfer_skcipher_request_to_engine(engine, areq);392}393 394int sun8i_ce_skencrypt(struct skcipher_request *areq)395{396 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);397 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);398 struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);399 struct crypto_engine *engine;400 int e;401 402 rctx->op_dir = CE_ENCRYPTION;403 if (sun8i_ce_cipher_need_fallback(areq))404 return sun8i_ce_cipher_fallback(areq);405 406 e = sun8i_ce_get_engine_number(op->ce);407 rctx->flow = e;408 engine = op->ce->chanlist[e].engine;409 410 return crypto_transfer_skcipher_request_to_engine(engine, areq);411}412 413int sun8i_ce_cipher_init(struct crypto_tfm *tfm)414{415 struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);416 struct sun8i_ce_alg_template *algt;417 const char *name = crypto_tfm_alg_name(tfm);418 struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);419 struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);420 int err;421 422 memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx));423 424 algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);425 op->ce = algt->ce;426 427 op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);428 if (IS_ERR(op->fallback_tfm)) {429 dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n",430 name, PTR_ERR(op->fallback_tfm));431 return PTR_ERR(op->fallback_tfm);432 }433 434 crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) +435 crypto_skcipher_reqsize(op->fallback_tfm));436 437 memcpy(algt->fbname,438 crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)),439 CRYPTO_MAX_ALG_NAME);440 441 err = pm_runtime_get_sync(op->ce->dev);442 if (err < 0)443 goto error_pm;444 445 return 0;446error_pm:447 pm_runtime_put_noidle(op->ce->dev);448 crypto_free_skcipher(op->fallback_tfm);449 return err;450}451 452void sun8i_ce_cipher_exit(struct crypto_tfm *tfm)453{454 struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);455 456 kfree_sensitive(op->key);457 crypto_free_skcipher(op->fallback_tfm);458 pm_runtime_put_sync_suspend(op->ce->dev);459}460 461int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,462 unsigned int keylen)463{464 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);465 struct sun8i_ce_dev *ce = op->ce;466 467 switch (keylen) {468 case 128 / 8:469 break;470 case 192 / 8:471 break;472 case 256 / 8:473 break;474 default:475 dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);476 return -EINVAL;477 }478 kfree_sensitive(op->key);479 op->keylen = keylen;480 op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);481 if (!op->key)482 return -ENOMEM;483 484 crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);485 crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);486 487 return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);488}489 490int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,491 unsigned int keylen)492{493 struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);494 int err;495 496 err = verify_skcipher_des3_key(tfm, key);497 if (err)498 return err;499 500 kfree_sensitive(op->key);501 op->keylen = keylen;502 op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);503 if (!op->key)504 return -ENOMEM;505 506 crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);507 crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);508 509 return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);510}511