389 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Cryptographic API.4 *5 * Support for OMAP AES GCM HW acceleration.6 *7 * Copyright (c) 2016 Texas Instruments Incorporated8 */9 10#include <crypto/aes.h>11#include <crypto/engine.h>12#include <crypto/gcm.h>13#include <crypto/internal/aead.h>14#include <crypto/scatterwalk.h>15#include <crypto/skcipher.h>16#include <linux/errno.h>17#include <linux/dma-mapping.h>18#include <linux/dmaengine.h>19#include <linux/interrupt.h>20#include <linux/kernel.h>21#include <linux/omap-dma.h>22#include <linux/pm_runtime.h>23#include <linux/scatterlist.h>24#include <linux/string.h>25 26#include "omap-crypto.h"27#include "omap-aes.h"28 29static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,30 struct aead_request *req);31 32static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)33{34 struct aead_request *req = dd->aead_req;35 36 dd->in_sg = NULL;37 dd->out_sg = NULL;38 39 crypto_finalize_aead_request(dd->engine, req, ret);40 41 pm_runtime_mark_last_busy(dd->dev);42 pm_runtime_put_autosuspend(dd->dev);43}44 45static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)46{47 u8 *tag;48 int alen, clen, i, ret = 0, nsg;49 struct omap_aes_reqctx *rctx;50 51 alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);52 clen = ALIGN(dd->total, AES_BLOCK_SIZE);53 rctx = aead_request_ctx(dd->aead_req);54 55 nsg = !!(dd->assoc_len && dd->total);56 57 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,58 DMA_FROM_DEVICE);59 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);60 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);61 omap_aes_crypt_dma_stop(dd);62 63 omap_crypto_cleanup(dd->out_sg, dd->orig_out,64 dd->aead_req->assoclen, dd->total,65 FLAGS_OUT_DATA_ST_SHIFT, dd->flags);66 67 if (dd->flags & FLAGS_ENCRYPT)68 scatterwalk_map_and_copy(rctx->auth_tag,69 dd->aead_req->dst,70 dd->total + dd->aead_req->assoclen,71 dd->authsize, 1);72 73 omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,74 FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);75 76 omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,77 FLAGS_IN_DATA_ST_SHIFT, dd->flags);78 79 if (!(dd->flags & FLAGS_ENCRYPT)) {80 tag = (u8 *)rctx->auth_tag;81 for (i = 0; i < dd->authsize; i++) {82 if (tag[i]) {83 ret = -EBADMSG;84 }85 }86 }87 88 omap_aes_gcm_finish_req(dd, ret);89}90 91static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,92 struct aead_request *req)93{94 int alen, clen, cryptlen, assoclen, ret;95 struct crypto_aead *aead = crypto_aead_reqtfm(req);96 unsigned int authlen = crypto_aead_authsize(aead);97 struct scatterlist *tmp, sg_arr[2];98 int nsg;99 u16 flags;100 101 assoclen = req->assoclen;102 cryptlen = req->cryptlen;103 104 if (dd->flags & FLAGS_RFC4106_GCM)105 assoclen -= 8;106 107 if (!(dd->flags & FLAGS_ENCRYPT))108 cryptlen -= authlen;109 110 alen = ALIGN(assoclen, AES_BLOCK_SIZE);111 clen = ALIGN(cryptlen, AES_BLOCK_SIZE);112 113 nsg = !!(assoclen && cryptlen);114 115 omap_aes_clear_copy_flags(dd);116 117 sg_init_table(dd->in_sgl, nsg + 1);118 if (assoclen) {119 tmp = req->src;120 ret = omap_crypto_align_sg(&tmp, assoclen,121 AES_BLOCK_SIZE, dd->in_sgl,122 OMAP_CRYPTO_COPY_DATA |123 OMAP_CRYPTO_ZERO_BUF |124 OMAP_CRYPTO_FORCE_SINGLE_ENTRY,125 FLAGS_ASSOC_DATA_ST_SHIFT,126 &dd->flags);127 if (ret)128 return ret;129 }130 131 if (cryptlen) {132 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);133 134 if (nsg)135 sg_unmark_end(dd->in_sgl);136 137 ret = omap_crypto_align_sg(&tmp, cryptlen,138 AES_BLOCK_SIZE, &dd->in_sgl[nsg],139 OMAP_CRYPTO_COPY_DATA |140 OMAP_CRYPTO_ZERO_BUF |141 OMAP_CRYPTO_FORCE_SINGLE_ENTRY,142 FLAGS_IN_DATA_ST_SHIFT,143 &dd->flags);144 if (ret)145 return ret;146 }147 148 dd->in_sg = dd->in_sgl;149 dd->total = cryptlen;150 dd->assoc_len = assoclen;151 dd->authsize = authlen;152 153 dd->out_sg = req->dst;154 dd->orig_out = req->dst;155 156 dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);157 158 flags = 0;159 if (req->src == req->dst || dd->out_sg == sg_arr)160 flags |= OMAP_CRYPTO_FORCE_COPY;161 162 if (cryptlen) {163 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,164 AES_BLOCK_SIZE, &dd->out_sgl,165 flags,166 FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);167 if (ret)168 return ret;169 }170 171 dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);172 dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);173 174 return 0;175}176 177static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)178{179 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));180 181 aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);182 return 0;183}184 185void omap_aes_gcm_dma_out_callback(void *data)186{187 struct omap_aes_dev *dd = data;188 struct omap_aes_reqctx *rctx;189 int i, val;190 u32 *auth_tag, tag[4];191 192 if (!(dd->flags & FLAGS_ENCRYPT))193 scatterwalk_map_and_copy(tag, dd->aead_req->src,194 dd->total + dd->aead_req->assoclen,195 dd->authsize, 0);196 197 rctx = aead_request_ctx(dd->aead_req);198 auth_tag = (u32 *)rctx->auth_tag;199 for (i = 0; i < 4; i++) {200 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));201 auth_tag[i] = val ^ auth_tag[i];202 if (!(dd->flags & FLAGS_ENCRYPT))203 auth_tag[i] = auth_tag[i] ^ tag[i];204 }205 206 omap_aes_gcm_done_task(dd);207}208 209static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,210 struct aead_request *req)211{212 if (req)213 return crypto_transfer_aead_request_to_engine(dd->engine, req);214 215 return 0;216}217 218static int omap_aes_gcm_prepare_req(struct aead_request *req,219 struct omap_aes_dev *dd)220{221 struct omap_aes_reqctx *rctx = aead_request_ctx(req);222 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));223 int err;224 225 dd->aead_req = req;226 227 rctx->mode &= FLAGS_MODE_MASK;228 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;229 230 err = omap_aes_gcm_copy_buffers(dd, req);231 if (err)232 return err;233 234 dd->ctx = &ctx->octx;235 236 return omap_aes_write_ctrl(dd);237}238 239static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)240{241 struct omap_aes_reqctx *rctx = aead_request_ctx(req);242 struct crypto_aead *aead = crypto_aead_reqtfm(req);243 unsigned int authlen = crypto_aead_authsize(aead);244 struct omap_aes_dev *dd;245 __be32 counter = cpu_to_be32(1);246 int err, assoclen;247 248 memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));249 memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);250 251 err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);252 if (err)253 return err;254 255 if (mode & FLAGS_RFC4106_GCM)256 assoclen = req->assoclen - 8;257 else258 assoclen = req->assoclen;259 if (assoclen + req->cryptlen == 0) {260 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,261 1);262 return 0;263 }264 265 dd = omap_aes_find_dev(rctx);266 if (!dd)267 return -ENODEV;268 rctx->mode = mode;269 270 return omap_aes_gcm_handle_queue(dd, req);271}272 273int omap_aes_gcm_encrypt(struct aead_request *req)274{275 struct omap_aes_reqctx *rctx = aead_request_ctx(req);276 277 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);278 return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);279}280 281int omap_aes_gcm_decrypt(struct aead_request *req)282{283 struct omap_aes_reqctx *rctx = aead_request_ctx(req);284 285 memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);286 return omap_aes_gcm_crypt(req, FLAGS_GCM);287}288 289int omap_aes_4106gcm_encrypt(struct aead_request *req)290{291 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));292 struct omap_aes_reqctx *rctx = aead_request_ctx(req);293 294 memcpy(rctx->iv, ctx->octx.nonce, 4);295 memcpy(rctx->iv + 4, req->iv, 8);296 return crypto_ipsec_check_assoclen(req->assoclen) ?:297 omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |298 FLAGS_RFC4106_GCM);299}300 301int omap_aes_4106gcm_decrypt(struct aead_request *req)302{303 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));304 struct omap_aes_reqctx *rctx = aead_request_ctx(req);305 306 memcpy(rctx->iv, ctx->octx.nonce, 4);307 memcpy(rctx->iv + 4, req->iv, 8);308 return crypto_ipsec_check_assoclen(req->assoclen) ?:309 omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);310}311 312int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,313 unsigned int keylen)314{315 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);316 int ret;317 318 ret = aes_expandkey(&ctx->actx, key, keylen);319 if (ret)320 return ret;321 322 memcpy(ctx->octx.key, key, keylen);323 ctx->octx.keylen = keylen;324 325 return 0;326}327 328int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,329 unsigned int keylen)330{331 struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);332 int ret;333 334 if (keylen < 4)335 return -EINVAL;336 keylen -= 4;337 338 ret = aes_expandkey(&ctx->actx, key, keylen);339 if (ret)340 return ret;341 342 memcpy(ctx->octx.key, key, keylen);343 memcpy(ctx->octx.nonce, key + keylen, 4);344 ctx->octx.keylen = keylen;345 346 return 0;347}348 349int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)350{351 return crypto_gcm_check_authsize(authsize);352}353 354int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,355 unsigned int authsize)356{357 return crypto_rfc4106_check_authsize(authsize);358}359 360int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)361{362 struct aead_request *req = container_of(areq, struct aead_request,363 base);364 struct omap_aes_reqctx *rctx = aead_request_ctx(req);365 struct omap_aes_dev *dd = rctx->dd;366 int ret;367 368 if (!dd)369 return -ENODEV;370 371 ret = omap_aes_gcm_prepare_req(req, dd);372 if (ret)373 return ret;374 375 if (dd->in_sg_len)376 ret = omap_aes_crypt_dma_start(dd);377 else378 omap_aes_gcm_dma_out_callback(dd);379 380 return ret;381}382 383int omap_aes_gcm_cra_init(struct crypto_aead *tfm)384{385 crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));386 387 return 0;388}389