390 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * sl3516-ce-cipher.c - hardware cryptographic offloader for Storlink SL3516 SoC4 *5 * Copyright (C) 2021 Corentin LABBE <clabbe@baylibre.com>6 *7 * This file adds support for AES cipher with 128,192,256 bits keysize in8 * ECB mode.9 */10 11#include <crypto/engine.h>12#include <crypto/internal/skcipher.h>13#include <crypto/scatterwalk.h>14#include <linux/dma-mapping.h>15#include <linux/delay.h>16#include <linux/err.h>17#include <linux/io.h>18#include <linux/kernel.h>19#include <linux/pm_runtime.h>20#include <linux/slab.h>21#include <linux/string.h>22#include "sl3516-ce.h"23 24/* sl3516_ce_need_fallback - check if a request can be handled by the CE */25static bool sl3516_ce_need_fallback(struct skcipher_request *areq)26{27 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);28 struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);29 struct sl3516_ce_dev *ce = op->ce;30 struct scatterlist *in_sg;31 struct scatterlist *out_sg;32 struct scatterlist *sg;33 34 if (areq->cryptlen == 0 || areq->cryptlen % 16) {35 ce->fallback_mod16++;36 return true;37 }38 39 /*40 * check if we have enough descriptors for TX41 * Note: TX need one control desc for each SG42 */43 if (sg_nents(areq->src) > MAXDESC / 2) {44 ce->fallback_sg_count_tx++;45 return true;46 }47 /* check if we have enough descriptors for RX */48 if (sg_nents(areq->dst) > MAXDESC) {49 ce->fallback_sg_count_rx++;50 return true;51 }52 53 sg = areq->src;54 while (sg) {55 if ((sg->length % 16) != 0) {56 ce->fallback_mod16++;57 return true;58 }59 if ((sg_dma_len(sg) % 16) != 0) {60 ce->fallback_mod16++;61 return true;62 }63 if (!IS_ALIGNED(sg->offset, 16)) {64 ce->fallback_align16++;65 return true;66 }67 sg = sg_next(sg);68 }69 sg = areq->dst;70 while (sg) {71 if ((sg->length % 16) != 0) {72 ce->fallback_mod16++;73 return true;74 }75 if ((sg_dma_len(sg) % 16) != 0) {76 ce->fallback_mod16++;77 return true;78 }79 if (!IS_ALIGNED(sg->offset, 16)) {80 ce->fallback_align16++;81 return true;82 }83 sg = sg_next(sg);84 }85 86 /* need same numbers of SG (with same length) for source and destination */87 in_sg = areq->src;88 out_sg = areq->dst;89 while (in_sg && out_sg) {90 if (in_sg->length != out_sg->length) {91 ce->fallback_not_same_len++;92 return true;93 }94 in_sg = sg_next(in_sg);95 out_sg = sg_next(out_sg);96 }97 if (in_sg || out_sg)98 return true;99 100 return false;101}102 103static int sl3516_ce_cipher_fallback(struct skcipher_request *areq)104{105 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);106 struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);107 struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);108 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);109 struct sl3516_ce_alg_template *algt;110 int err;111 112 algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base);113 algt->stat_fb++;114 115 skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);116 skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,117 areq->base.complete, areq->base.data);118 skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,119 areq->cryptlen, areq->iv);120 if (rctx->op_dir == CE_DECRYPTION)121 err = crypto_skcipher_decrypt(&rctx->fallback_req);122 else123 err = crypto_skcipher_encrypt(&rctx->fallback_req);124 return err;125}126 127static int sl3516_ce_cipher(struct skcipher_request *areq)128{129 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);130 struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);131 struct sl3516_ce_dev *ce = op->ce;132 struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);133 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);134 struct sl3516_ce_alg_template *algt;135 struct scatterlist *sg;136 unsigned int todo, len;137 struct pkt_control_ecb *ecb;138 int nr_sgs = 0;139 int nr_sgd = 0;140 int err = 0;141 int i;142 143 algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base);144 145 dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,146 crypto_tfm_alg_name(areq->base.tfm),147 areq->cryptlen,148 rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),149 op->keylen);150 151 algt->stat_req++;152 153 if (areq->src == areq->dst) {154 nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src),155 DMA_BIDIRECTIONAL);156 if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) {157 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);158 err = -EINVAL;159 goto theend;160 }161 nr_sgd = nr_sgs;162 } else {163 nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src),164 DMA_TO_DEVICE);165 if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) {166 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);167 err = -EINVAL;168 goto theend;169 }170 nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst),171 DMA_FROM_DEVICE);172 if (nr_sgd <= 0 || nr_sgd > MAXDESC) {173 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd);174 err = -EINVAL;175 goto theend_sgs;176 }177 }178 179 len = areq->cryptlen;180 i = 0;181 sg = areq->src;182 while (i < nr_sgs && sg && len) {183 if (sg_dma_len(sg) == 0)184 goto sgs_next;185 rctx->t_src[i].addr = sg_dma_address(sg);186 todo = min(len, sg_dma_len(sg));187 rctx->t_src[i].len = todo;188 dev_dbg(ce->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__,189 areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo);190 len -= todo;191 i++;192sgs_next:193 sg = sg_next(sg);194 }195 if (len > 0) {196 dev_err(ce->dev, "remaining len %d/%u nr_sgs=%d\n", len, areq->cryptlen, nr_sgs);197 err = -EINVAL;198 goto theend_sgs;199 }200 201 len = areq->cryptlen;202 i = 0;203 sg = areq->dst;204 while (i < nr_sgd && sg && len) {205 if (sg_dma_len(sg) == 0)206 goto sgd_next;207 rctx->t_dst[i].addr = sg_dma_address(sg);208 todo = min(len, sg_dma_len(sg));209 rctx->t_dst[i].len = todo;210 dev_dbg(ce->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__,211 areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo);212 len -= todo;213 i++;214 215sgd_next:216 sg = sg_next(sg);217 }218 if (len > 0) {219 dev_err(ce->dev, "remaining len %d\n", len);220 err = -EINVAL;221 goto theend_sgs;222 }223 224 switch (algt->mode) {225 case ECB_AES:226 rctx->pctrllen = sizeof(struct pkt_control_ecb);227 ecb = (struct pkt_control_ecb *)ce->pctrl;228 229 rctx->tqflag = TQ0_TYPE_CTRL;230 rctx->tqflag |= TQ1_CIPHER;231 ecb->control.op_mode = rctx->op_dir;232 ecb->control.cipher_algorithm = ECB_AES;233 ecb->cipher.header_len = 0;234 ecb->cipher.algorithm_len = areq->cryptlen;235 cpu_to_be32_array((__be32 *)ecb->key, (u32 *)op->key, op->keylen / 4);236 rctx->h = &ecb->cipher;237 238 rctx->tqflag |= TQ4_KEY0;239 rctx->tqflag |= TQ5_KEY4;240 rctx->tqflag |= TQ6_KEY6;241 ecb->control.aesnk = op->keylen / 4;242 break;243 }244 245 rctx->nr_sgs = nr_sgs;246 rctx->nr_sgd = nr_sgd;247 err = sl3516_ce_run_task(ce, rctx, crypto_tfm_alg_name(areq->base.tfm));248 249theend_sgs:250 if (areq->src == areq->dst) {251 dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src),252 DMA_BIDIRECTIONAL);253 } else {254 dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src),255 DMA_TO_DEVICE);256 dma_unmap_sg(ce->dev, areq->dst, sg_nents(areq->dst),257 DMA_FROM_DEVICE);258 }259 260theend:261 262 return err;263}264 265int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq)266{267 int err;268 struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);269 270 err = sl3516_ce_cipher(breq);271 local_bh_disable();272 crypto_finalize_skcipher_request(engine, breq, err);273 local_bh_enable();274 275 return 0;276}277 278int sl3516_ce_skdecrypt(struct skcipher_request *areq)279{280 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);281 struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);282 struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);283 struct crypto_engine *engine;284 285 memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx));286 rctx->op_dir = CE_DECRYPTION;287 288 if (sl3516_ce_need_fallback(areq))289 return sl3516_ce_cipher_fallback(areq);290 291 engine = op->ce->engine;292 293 return crypto_transfer_skcipher_request_to_engine(engine, areq);294}295 296int sl3516_ce_skencrypt(struct skcipher_request *areq)297{298 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);299 struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);300 struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq);301 struct crypto_engine *engine;302 303 memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx));304 rctx->op_dir = CE_ENCRYPTION;305 306 if (sl3516_ce_need_fallback(areq))307 return sl3516_ce_cipher_fallback(areq);308 309 engine = op->ce->engine;310 311 return crypto_transfer_skcipher_request_to_engine(engine, areq);312}313 314int sl3516_ce_cipher_init(struct crypto_tfm *tfm)315{316 struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);317 struct sl3516_ce_alg_template *algt;318 const char *name = crypto_tfm_alg_name(tfm);319 struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);320 struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);321 int err;322 323 memset(op, 0, sizeof(struct sl3516_ce_cipher_tfm_ctx));324 325 algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base);326 op->ce = algt->ce;327 328 op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);329 if (IS_ERR(op->fallback_tfm)) {330 dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n",331 name, PTR_ERR(op->fallback_tfm));332 return PTR_ERR(op->fallback_tfm);333 }334 335 crypto_skcipher_set_reqsize(sktfm, sizeof(struct sl3516_ce_cipher_req_ctx) +336 crypto_skcipher_reqsize(op->fallback_tfm));337 338 dev_info(op->ce->dev, "Fallback for %s is %s\n",339 crypto_tfm_alg_driver_name(&sktfm->base),340 crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)));341 342 err = pm_runtime_get_sync(op->ce->dev);343 if (err < 0)344 goto error_pm;345 346 return 0;347error_pm:348 pm_runtime_put_noidle(op->ce->dev);349 crypto_free_skcipher(op->fallback_tfm);350 return err;351}352 353void sl3516_ce_cipher_exit(struct crypto_tfm *tfm)354{355 struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);356 357 kfree_sensitive(op->key);358 crypto_free_skcipher(op->fallback_tfm);359 pm_runtime_put_sync_suspend(op->ce->dev);360}361 362int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,363 unsigned int keylen)364{365 struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);366 struct sl3516_ce_dev *ce = op->ce;367 368 switch (keylen) {369 case 128 / 8:370 break;371 case 192 / 8:372 break;373 case 256 / 8:374 break;375 default:376 dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);377 return -EINVAL;378 }379 kfree_sensitive(op->key);380 op->keylen = keylen;381 op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);382 if (!op->key)383 return -ENOMEM;384 385 crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);386 crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);387 388 return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);389}390