479 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * sun8i-ce-hash.c - hardware cryptographic offloader for4 * Allwinner H3/A64/H5/H2+/H6/R40 SoC5 *6 * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>7 *8 * This file add support for MD5 and SHA1/SHA224/SHA256/SHA384/SHA512.9 *10 * You could find the datasheet in Documentation/arch/arm/sunxi.rst11 */12 13#include <crypto/internal/hash.h>14#include <crypto/md5.h>15#include <crypto/sha1.h>16#include <crypto/sha2.h>17#include <linux/bottom_half.h>18#include <linux/dma-mapping.h>19#include <linux/kernel.h>20#include <linux/pm_runtime.h>21#include <linux/scatterlist.h>22#include <linux/slab.h>23#include <linux/string.h>24#include "sun8i-ce.h"25 26int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm)27{28 struct sun8i_ce_hash_tfm_ctx *op = crypto_ahash_ctx(tfm);29 struct ahash_alg *alg = crypto_ahash_alg(tfm);30 struct sun8i_ce_alg_template *algt;31 int err;32 33 algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);34 op->ce = algt->ce;35 36 /* FALLBACK */37 op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0,38 CRYPTO_ALG_NEED_FALLBACK);39 if (IS_ERR(op->fallback_tfm)) {40 dev_err(algt->ce->dev, "Fallback driver could no be loaded\n");41 return PTR_ERR(op->fallback_tfm);42 }43 44 crypto_ahash_set_statesize(tfm,45 crypto_ahash_statesize(op->fallback_tfm));46 47 crypto_ahash_set_reqsize(tfm,48 sizeof(struct sun8i_ce_hash_reqctx) +49 crypto_ahash_reqsize(op->fallback_tfm));50 51 memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm),52 CRYPTO_MAX_ALG_NAME);53 54 err = pm_runtime_get_sync(op->ce->dev);55 if (err < 0)56 goto error_pm;57 return 0;58error_pm:59 pm_runtime_put_noidle(op->ce->dev);60 crypto_free_ahash(op->fallback_tfm);61 return err;62}63 64void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm)65{66 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);67 68 crypto_free_ahash(tfmctx->fallback_tfm);69 pm_runtime_put_sync_suspend(tfmctx->ce->dev);70}71 72int sun8i_ce_hash_init(struct ahash_request *areq)73{74 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);75 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);76 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);77 78 memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx));79 80 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);81 rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;82 83 return crypto_ahash_init(&rctx->fallback_req);84}85 86int sun8i_ce_hash_export(struct ahash_request *areq, void *out)87{88 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);89 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);90 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);91 92 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);93 rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;94 95 return crypto_ahash_export(&rctx->fallback_req, out);96}97 98int sun8i_ce_hash_import(struct ahash_request *areq, const void *in)99{100 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);101 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);102 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);103 104 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);105 rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;106 107 return crypto_ahash_import(&rctx->fallback_req, in);108}109 110int sun8i_ce_hash_final(struct ahash_request *areq)111{112 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);113 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);114 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);115 116 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);117 rctx->fallback_req.base.flags = areq->base.flags &118 CRYPTO_TFM_REQ_MAY_SLEEP;119 rctx->fallback_req.result = areq->result;120 121 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {122 struct sun8i_ce_alg_template *algt __maybe_unused;123 struct ahash_alg *alg = crypto_ahash_alg(tfm);124 125 algt = container_of(alg, struct sun8i_ce_alg_template,126 alg.hash.base);127#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG128 algt->stat_fb++;129#endif130 }131 132 return crypto_ahash_final(&rctx->fallback_req);133}134 135int sun8i_ce_hash_update(struct ahash_request *areq)136{137 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);138 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);139 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);140 141 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);142 rctx->fallback_req.base.flags = areq->base.flags &143 CRYPTO_TFM_REQ_MAY_SLEEP;144 rctx->fallback_req.nbytes = areq->nbytes;145 rctx->fallback_req.src = areq->src;146 147 return crypto_ahash_update(&rctx->fallback_req);148}149 150int sun8i_ce_hash_finup(struct ahash_request *areq)151{152 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);153 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);154 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);155 156 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);157 rctx->fallback_req.base.flags = areq->base.flags &158 CRYPTO_TFM_REQ_MAY_SLEEP;159 160 rctx->fallback_req.nbytes = areq->nbytes;161 rctx->fallback_req.src = areq->src;162 rctx->fallback_req.result = areq->result;163 164 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {165 struct sun8i_ce_alg_template *algt __maybe_unused;166 struct ahash_alg *alg = crypto_ahash_alg(tfm);167 168 algt = container_of(alg, struct sun8i_ce_alg_template,169 alg.hash.base);170#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG171 algt->stat_fb++;172#endif173 }174 175 return crypto_ahash_finup(&rctx->fallback_req);176}177 178static int sun8i_ce_hash_digest_fb(struct ahash_request *areq)179{180 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);181 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);182 struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);183 184 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);185 rctx->fallback_req.base.flags = areq->base.flags &186 CRYPTO_TFM_REQ_MAY_SLEEP;187 188 rctx->fallback_req.nbytes = areq->nbytes;189 rctx->fallback_req.src = areq->src;190 rctx->fallback_req.result = areq->result;191 192 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {193 struct sun8i_ce_alg_template *algt __maybe_unused;194 struct ahash_alg *alg = crypto_ahash_alg(tfm);195 196 algt = container_of(alg, struct sun8i_ce_alg_template,197 alg.hash.base);198#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG199 algt->stat_fb++;200#endif201 }202 203 return crypto_ahash_digest(&rctx->fallback_req);204}205 206static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq)207{208 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);209 struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);210 struct sun8i_ce_alg_template *algt;211 struct scatterlist *sg;212 213 algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);214 215 if (areq->nbytes == 0) {216 algt->stat_fb_len0++;217 return true;218 }219 /* we need to reserve one SG for padding one */220 if (sg_nents_for_len(areq->src, areq->nbytes) > MAX_SG - 1) {221 algt->stat_fb_maxsg++;222 return true;223 }224 sg = areq->src;225 while (sg) {226 if (sg->length % 4) {227 algt->stat_fb_srclen++;228 return true;229 }230 if (!IS_ALIGNED(sg->offset, sizeof(u32))) {231 algt->stat_fb_srcali++;232 return true;233 }234 sg = sg_next(sg);235 }236 return false;237}238 239int sun8i_ce_hash_digest(struct ahash_request *areq)240{241 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);242 struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);243 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);244 struct sun8i_ce_alg_template *algt;245 struct sun8i_ce_dev *ce;246 struct crypto_engine *engine;247 struct scatterlist *sg;248 int nr_sgs, e, i;249 250 if (sun8i_ce_hash_need_fallback(areq))251 return sun8i_ce_hash_digest_fb(areq);252 253 nr_sgs = sg_nents_for_len(areq->src, areq->nbytes);254 if (nr_sgs > MAX_SG - 1)255 return sun8i_ce_hash_digest_fb(areq);256 257 for_each_sg(areq->src, sg, nr_sgs, i) {258 if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32)))259 return sun8i_ce_hash_digest_fb(areq);260 }261 262 algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);263 ce = algt->ce;264 265 e = sun8i_ce_get_engine_number(ce);266 rctx->flow = e;267 engine = ce->chanlist[e].engine;268 269 return crypto_transfer_hash_request_to_engine(engine, areq);270}271 272static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs)273{274 u64 fill, min_fill, j, k;275 __be64 *bebits;276 __le64 *lebits;277 278 j = padi;279 buf[j++] = cpu_to_le32(0x80);280 281 if (bs == 64) {282 fill = 64 - (byte_count % 64);283 min_fill = 2 * sizeof(u32) + sizeof(u32);284 } else {285 fill = 128 - (byte_count % 128);286 min_fill = 4 * sizeof(u32) + sizeof(u32);287 }288 289 if (fill < min_fill)290 fill += bs;291 292 k = j;293 j += (fill - min_fill) / sizeof(u32);294 if (j * 4 > bufsize) {295 pr_err("%s OVERFLOW %llu\n", __func__, j);296 return 0;297 }298 for (; k < j; k++)299 buf[k] = 0;300 301 if (le) {302 /* MD5 */303 lebits = (__le64 *)&buf[j];304 *lebits = cpu_to_le64(byte_count << 3);305 j += 2;306 } else {307 if (bs == 64) {308 /* sha1 sha224 sha256 */309 bebits = (__be64 *)&buf[j];310 *bebits = cpu_to_be64(byte_count << 3);311 j += 2;312 } else {313 /* sha384 sha512*/314 bebits = (__be64 *)&buf[j];315 *bebits = cpu_to_be64(byte_count >> 61);316 j += 2;317 bebits = (__be64 *)&buf[j];318 *bebits = cpu_to_be64(byte_count << 3);319 j += 2;320 }321 }322 if (j * 4 > bufsize) {323 pr_err("%s OVERFLOW %llu\n", __func__, j);324 return 0;325 }326 327 return j;328}329 330int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq)331{332 struct ahash_request *areq = container_of(breq, struct ahash_request, base);333 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);334 struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);335 struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);336 struct sun8i_ce_alg_template *algt;337 struct sun8i_ce_dev *ce;338 struct sun8i_ce_flow *chan;339 struct ce_task *cet;340 struct scatterlist *sg;341 int nr_sgs, flow, err;342 unsigned int len;343 u32 common;344 u64 byte_count;345 __le32 *bf;346 void *buf = NULL;347 int j, i, todo;348 void *result = NULL;349 u64 bs;350 int digestsize;351 dma_addr_t addr_res, addr_pad;352 int ns = sg_nents_for_len(areq->src, areq->nbytes);353 354 algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);355 ce = algt->ce;356 357 bs = algt->alg.hash.base.halg.base.cra_blocksize;358 digestsize = algt->alg.hash.base.halg.digestsize;359 if (digestsize == SHA224_DIGEST_SIZE)360 digestsize = SHA256_DIGEST_SIZE;361 if (digestsize == SHA384_DIGEST_SIZE)362 digestsize = SHA512_DIGEST_SIZE;363 364 /* the padding could be up to two block. */365 buf = kcalloc(2, bs, GFP_KERNEL | GFP_DMA);366 if (!buf) {367 err = -ENOMEM;368 goto theend;369 }370 bf = (__le32 *)buf;371 372 result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA);373 if (!result) {374 err = -ENOMEM;375 goto theend;376 }377 378 flow = rctx->flow;379 chan = &ce->chanlist[flow];380 381#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG382 algt->stat_req++;383#endif384 dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes);385 386 cet = chan->tl;387 memset(cet, 0, sizeof(struct ce_task));388 389 cet->t_id = cpu_to_le32(flow);390 common = ce->variant->alg_hash[algt->ce_algo_id];391 common |= CE_COMM_INT;392 cet->t_common_ctl = cpu_to_le32(common);393 394 cet->t_sym_ctl = 0;395 cet->t_asym_ctl = 0;396 397 nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);398 if (nr_sgs <= 0 || nr_sgs > MAX_SG) {399 dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);400 err = -EINVAL;401 goto theend;402 }403 404 len = areq->nbytes;405 for_each_sg(areq->src, sg, nr_sgs, i) {406 cet->t_src[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg));407 todo = min(len, sg_dma_len(sg));408 cet->t_src[i].len = cpu_to_le32(todo / 4);409 len -= todo;410 }411 if (len > 0) {412 dev_err(ce->dev, "remaining len %d\n", len);413 err = -EINVAL;414 goto theend;415 }416 addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE);417 cet->t_dst[0].addr = desc_addr_val_le32(ce, addr_res);418 cet->t_dst[0].len = cpu_to_le32(digestsize / 4);419 if (dma_mapping_error(ce->dev, addr_res)) {420 dev_err(ce->dev, "DMA map dest\n");421 err = -EINVAL;422 goto theend;423 }424 425 byte_count = areq->nbytes;426 j = 0;427 428 switch (algt->ce_algo_id) {429 case CE_ID_HASH_MD5:430 j = hash_pad(bf, 2 * bs, j, byte_count, true, bs);431 break;432 case CE_ID_HASH_SHA1:433 case CE_ID_HASH_SHA224:434 case CE_ID_HASH_SHA256:435 j = hash_pad(bf, 2 * bs, j, byte_count, false, bs);436 break;437 case CE_ID_HASH_SHA384:438 case CE_ID_HASH_SHA512:439 j = hash_pad(bf, 2 * bs, j, byte_count, false, bs);440 break;441 }442 if (!j) {443 err = -EINVAL;444 goto theend;445 }446 447 addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE);448 cet->t_src[i].addr = desc_addr_val_le32(ce, addr_pad);449 cet->t_src[i].len = cpu_to_le32(j);450 if (dma_mapping_error(ce->dev, addr_pad)) {451 dev_err(ce->dev, "DMA error on padding SG\n");452 err = -EINVAL;453 goto theend;454 }455 456 if (ce->variant->hash_t_dlen_in_bits)457 cet->t_dlen = cpu_to_le32((areq->nbytes + j * 4) * 8);458 else459 cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j);460 461 chan->timeout = areq->nbytes;462 463 err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm));464 465 dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE);466 dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);467 dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE);468 469 470 memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize);471theend:472 kfree(buf);473 kfree(result);474 local_bh_disable();475 crypto_finalize_hash_request(engine, breq, err);476 local_bh_enable();477 return 0;478}479