2706 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Cryptographic API.4 *5 * Support for ATMEL SHA1/SHA256 HW acceleration.6 *7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL8 * Author: Nicolas Royer <nicolas@eukrea.com>9 *10 * Some ideas are from omap-sham.c drivers.11 */12 13 14#include <linux/kernel.h>15#include <linux/module.h>16#include <linux/slab.h>17#include <linux/err.h>18#include <linux/clk.h>19#include <linux/io.h>20#include <linux/hw_random.h>21#include <linux/platform_device.h>22 23#include <linux/device.h>24#include <linux/dmaengine.h>25#include <linux/init.h>26#include <linux/errno.h>27#include <linux/interrupt.h>28#include <linux/irq.h>29#include <linux/scatterlist.h>30#include <linux/dma-mapping.h>31#include <linux/mod_devicetable.h>32#include <linux/delay.h>33#include <linux/crypto.h>34#include <crypto/scatterwalk.h>35#include <crypto/algapi.h>36#include <crypto/sha1.h>37#include <crypto/sha2.h>38#include <crypto/hash.h>39#include <crypto/internal/hash.h>40#include "atmel-sha-regs.h"41#include "atmel-authenc.h"42 43#define ATMEL_SHA_PRIORITY 30044 45/* SHA flags */46#define SHA_FLAGS_BUSY BIT(0)47#define SHA_FLAGS_FINAL BIT(1)48#define SHA_FLAGS_DMA_ACTIVE BIT(2)49#define SHA_FLAGS_OUTPUT_READY BIT(3)50#define SHA_FLAGS_INIT BIT(4)51#define SHA_FLAGS_CPU BIT(5)52#define SHA_FLAGS_DMA_READY BIT(6)53#define SHA_FLAGS_DUMP_REG BIT(7)54 55/* bits[11:8] are reserved. */56 57#define SHA_FLAGS_FINUP BIT(16)58#define SHA_FLAGS_SG BIT(17)59#define SHA_FLAGS_ERROR BIT(23)60#define SHA_FLAGS_PAD BIT(24)61#define SHA_FLAGS_RESTORE BIT(25)62#define SHA_FLAGS_IDATAR0 BIT(26)63#define SHA_FLAGS_WAIT_DATARDY BIT(27)64 65#define SHA_OP_INIT 066#define SHA_OP_UPDATE 167#define SHA_OP_FINAL 268#define SHA_OP_DIGEST 369 70#define SHA_BUFFER_LEN (PAGE_SIZE / 16)71 72#define ATMEL_SHA_DMA_THRESHOLD 5673 74struct atmel_sha_caps {75 bool has_dma;76 bool has_dualbuff;77 bool has_sha224;78 bool has_sha_384_512;79 bool has_uihv;80 bool has_hmac;81};82 83struct atmel_sha_dev;84 85/*86 * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as87 * tested by the ahash_prepare_alg() function.88 */89struct atmel_sha_reqctx {90 struct atmel_sha_dev *dd;91 unsigned long flags;92 unsigned long op;93 94 u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));95 u64 digcnt[2];96 size_t bufcnt;97 size_t buflen;98 dma_addr_t dma_addr;99 100 /* walk state */101 struct scatterlist *sg;102 unsigned int offset; /* offset in current sg */103 unsigned int total; /* total request */104 105 size_t block_size;106 size_t hash_size;107 108 u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32));109};110 111typedef int (*atmel_sha_fn_t)(struct atmel_sha_dev *);112 113struct atmel_sha_ctx {114 struct atmel_sha_dev *dd;115 atmel_sha_fn_t start;116 117 unsigned long flags;118};119 120#define ATMEL_SHA_QUEUE_LENGTH 50121 122struct atmel_sha_dma {123 struct dma_chan *chan;124 struct dma_slave_config dma_conf;125 struct scatterlist *sg;126 int nents;127 unsigned int last_sg_length;128};129 130struct atmel_sha_dev {131 struct list_head list;132 unsigned long phys_base;133 struct device *dev;134 struct clk *iclk;135 int irq;136 void __iomem *io_base;137 138 spinlock_t lock;139 struct tasklet_struct done_task;140 struct tasklet_struct queue_task;141 142 unsigned long flags;143 struct crypto_queue queue;144 struct ahash_request *req;145 bool is_async;146 bool force_complete;147 atmel_sha_fn_t resume;148 atmel_sha_fn_t cpu_transfer_complete;149 150 struct atmel_sha_dma dma_lch_in;151 152 struct atmel_sha_caps caps;153 154 struct scatterlist tmp;155 156 u32 hw_version;157};158 159struct atmel_sha_drv {160 struct list_head dev_list;161 spinlock_t lock;162};163 164static struct atmel_sha_drv atmel_sha = {165 .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list),166 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock),167};168 169#ifdef VERBOSE_DEBUG170static const char *atmel_sha_reg_name(u32 offset, char *tmp, size_t sz, bool wr)171{172 switch (offset) {173 case SHA_CR:174 return "CR";175 176 case SHA_MR:177 return "MR";178 179 case SHA_IER:180 return "IER";181 182 case SHA_IDR:183 return "IDR";184 185 case SHA_IMR:186 return "IMR";187 188 case SHA_ISR:189 return "ISR";190 191 case SHA_MSR:192 return "MSR";193 194 case SHA_BCR:195 return "BCR";196 197 case SHA_REG_DIN(0):198 case SHA_REG_DIN(1):199 case SHA_REG_DIN(2):200 case SHA_REG_DIN(3):201 case SHA_REG_DIN(4):202 case SHA_REG_DIN(5):203 case SHA_REG_DIN(6):204 case SHA_REG_DIN(7):205 case SHA_REG_DIN(8):206 case SHA_REG_DIN(9):207 case SHA_REG_DIN(10):208 case SHA_REG_DIN(11):209 case SHA_REG_DIN(12):210 case SHA_REG_DIN(13):211 case SHA_REG_DIN(14):212 case SHA_REG_DIN(15):213 snprintf(tmp, sz, "IDATAR[%u]", (offset - SHA_REG_DIN(0)) >> 2);214 break;215 216 case SHA_REG_DIGEST(0):217 case SHA_REG_DIGEST(1):218 case SHA_REG_DIGEST(2):219 case SHA_REG_DIGEST(3):220 case SHA_REG_DIGEST(4):221 case SHA_REG_DIGEST(5):222 case SHA_REG_DIGEST(6):223 case SHA_REG_DIGEST(7):224 case SHA_REG_DIGEST(8):225 case SHA_REG_DIGEST(9):226 case SHA_REG_DIGEST(10):227 case SHA_REG_DIGEST(11):228 case SHA_REG_DIGEST(12):229 case SHA_REG_DIGEST(13):230 case SHA_REG_DIGEST(14):231 case SHA_REG_DIGEST(15):232 if (wr)233 snprintf(tmp, sz, "IDATAR[%u]",234 16u + ((offset - SHA_REG_DIGEST(0)) >> 2));235 else236 snprintf(tmp, sz, "ODATAR[%u]",237 (offset - SHA_REG_DIGEST(0)) >> 2);238 break;239 240 case SHA_HW_VERSION:241 return "HWVER";242 243 default:244 snprintf(tmp, sz, "0x%02x", offset);245 break;246 }247 248 return tmp;249}250 251#endif /* VERBOSE_DEBUG */252 253static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset)254{255 u32 value = readl_relaxed(dd->io_base + offset);256 257#ifdef VERBOSE_DEBUG258 if (dd->flags & SHA_FLAGS_DUMP_REG) {259 char tmp[16];260 261 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,262 atmel_sha_reg_name(offset, tmp, sizeof(tmp), false));263 }264#endif /* VERBOSE_DEBUG */265 266 return value;267}268 269static inline void atmel_sha_write(struct atmel_sha_dev *dd,270 u32 offset, u32 value)271{272#ifdef VERBOSE_DEBUG273 if (dd->flags & SHA_FLAGS_DUMP_REG) {274 char tmp[16];275 276 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,277 atmel_sha_reg_name(offset, tmp, sizeof(tmp), true));278 }279#endif /* VERBOSE_DEBUG */280 281 writel_relaxed(value, dd->io_base + offset);282}283 284static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err)285{286 struct ahash_request *req = dd->req;287 288 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |289 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY |290 SHA_FLAGS_DUMP_REG);291 292 clk_disable(dd->iclk);293 294 if ((dd->is_async || dd->force_complete) && req->base.complete)295 ahash_request_complete(req, err);296 297 /* handle new request */298 tasklet_schedule(&dd->queue_task);299 300 return err;301}302 303static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)304{305 size_t count;306 307 while ((ctx->bufcnt < ctx->buflen) && ctx->total) {308 count = min(ctx->sg->length - ctx->offset, ctx->total);309 count = min(count, ctx->buflen - ctx->bufcnt);310 311 if (count <= 0) {312 /*313 * Check if count <= 0 because the buffer is full or314 * because the sg length is 0. In the latest case,315 * check if there is another sg in the list, a 0 length316 * sg doesn't necessarily mean the end of the sg list.317 */318 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {319 ctx->sg = sg_next(ctx->sg);320 continue;321 } else {322 break;323 }324 }325 326 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,327 ctx->offset, count, 0);328 329 ctx->bufcnt += count;330 ctx->offset += count;331 ctx->total -= count;332 333 if (ctx->offset == ctx->sg->length) {334 ctx->sg = sg_next(ctx->sg);335 if (ctx->sg)336 ctx->offset = 0;337 else338 ctx->total = 0;339 }340 }341 342 return 0;343}344 345/*346 * The purpose of this padding is to ensure that the padded message is a347 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).348 * The bit "1" is appended at the end of the message followed by349 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or350 * 128 bits block (SHA384/SHA512) equals to the message length in bits351 * is appended.352 *353 * For SHA1/SHA224/SHA256, padlen is calculated as followed:354 * - if message length < 56 bytes then padlen = 56 - message length355 * - else padlen = 64 + 56 - message length356 *357 * For SHA384/SHA512, padlen is calculated as followed:358 * - if message length < 112 bytes then padlen = 112 - message length359 * - else padlen = 128 + 112 - message length360 */361static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)362{363 unsigned int index, padlen;364 __be64 bits[2];365 u64 size[2];366 367 size[0] = ctx->digcnt[0];368 size[1] = ctx->digcnt[1];369 370 size[0] += ctx->bufcnt;371 if (size[0] < ctx->bufcnt)372 size[1]++;373 374 size[0] += length;375 if (size[0] < length)376 size[1]++;377 378 bits[1] = cpu_to_be64(size[0] << 3);379 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61);380 381 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {382 case SHA_FLAGS_SHA384:383 case SHA_FLAGS_SHA512:384 index = ctx->bufcnt & 0x7f;385 padlen = (index < 112) ? (112 - index) : ((128+112) - index);386 *(ctx->buffer + ctx->bufcnt) = 0x80;387 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);388 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);389 ctx->bufcnt += padlen + 16;390 ctx->flags |= SHA_FLAGS_PAD;391 break;392 393 default:394 index = ctx->bufcnt & 0x3f;395 padlen = (index < 56) ? (56 - index) : ((64+56) - index);396 *(ctx->buffer + ctx->bufcnt) = 0x80;397 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);398 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);399 ctx->bufcnt += padlen + 8;400 ctx->flags |= SHA_FLAGS_PAD;401 break;402 }403}404 405static struct atmel_sha_dev *atmel_sha_find_dev(struct atmel_sha_ctx *tctx)406{407 struct atmel_sha_dev *dd = NULL;408 struct atmel_sha_dev *tmp;409 410 spin_lock_bh(&atmel_sha.lock);411 if (!tctx->dd) {412 list_for_each_entry(tmp, &atmel_sha.dev_list, list) {413 dd = tmp;414 break;415 }416 tctx->dd = dd;417 } else {418 dd = tctx->dd;419 }420 421 spin_unlock_bh(&atmel_sha.lock);422 423 return dd;424}425 426static int atmel_sha_init(struct ahash_request *req)427{428 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);429 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);430 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);431 struct atmel_sha_dev *dd = atmel_sha_find_dev(tctx);432 433 ctx->dd = dd;434 435 ctx->flags = 0;436 437 dev_dbg(dd->dev, "init: digest size: %u\n",438 crypto_ahash_digestsize(tfm));439 440 switch (crypto_ahash_digestsize(tfm)) {441 case SHA1_DIGEST_SIZE:442 ctx->flags |= SHA_FLAGS_SHA1;443 ctx->block_size = SHA1_BLOCK_SIZE;444 break;445 case SHA224_DIGEST_SIZE:446 ctx->flags |= SHA_FLAGS_SHA224;447 ctx->block_size = SHA224_BLOCK_SIZE;448 break;449 case SHA256_DIGEST_SIZE:450 ctx->flags |= SHA_FLAGS_SHA256;451 ctx->block_size = SHA256_BLOCK_SIZE;452 break;453 case SHA384_DIGEST_SIZE:454 ctx->flags |= SHA_FLAGS_SHA384;455 ctx->block_size = SHA384_BLOCK_SIZE;456 break;457 case SHA512_DIGEST_SIZE:458 ctx->flags |= SHA_FLAGS_SHA512;459 ctx->block_size = SHA512_BLOCK_SIZE;460 break;461 default:462 return -EINVAL;463 }464 465 ctx->bufcnt = 0;466 ctx->digcnt[0] = 0;467 ctx->digcnt[1] = 0;468 ctx->buflen = SHA_BUFFER_LEN;469 470 return 0;471}472 473static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma)474{475 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);476 u32 valmr = SHA_MR_MODE_AUTO;477 unsigned int i, hashsize = 0;478 479 if (likely(dma)) {480 if (!dd->caps.has_dma)481 atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE);482 valmr = SHA_MR_MODE_PDC;483 if (dd->caps.has_dualbuff)484 valmr |= SHA_MR_DUALBUFF;485 } else {486 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);487 }488 489 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {490 case SHA_FLAGS_SHA1:491 valmr |= SHA_MR_ALGO_SHA1;492 hashsize = SHA1_DIGEST_SIZE;493 break;494 495 case SHA_FLAGS_SHA224:496 valmr |= SHA_MR_ALGO_SHA224;497 hashsize = SHA256_DIGEST_SIZE;498 break;499 500 case SHA_FLAGS_SHA256:501 valmr |= SHA_MR_ALGO_SHA256;502 hashsize = SHA256_DIGEST_SIZE;503 break;504 505 case SHA_FLAGS_SHA384:506 valmr |= SHA_MR_ALGO_SHA384;507 hashsize = SHA512_DIGEST_SIZE;508 break;509 510 case SHA_FLAGS_SHA512:511 valmr |= SHA_MR_ALGO_SHA512;512 hashsize = SHA512_DIGEST_SIZE;513 break;514 515 default:516 break;517 }518 519 /* Setting CR_FIRST only for the first iteration */520 if (!(ctx->digcnt[0] || ctx->digcnt[1])) {521 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);522 } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) {523 const u32 *hash = (const u32 *)ctx->digest;524 525 /*526 * Restore the hardware context: update the User Initialize527 * Hash Value (UIHV) with the value saved when the latest528 * 'update' operation completed on this very same crypto529 * request.530 */531 ctx->flags &= ~SHA_FLAGS_RESTORE;532 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);533 for (i = 0; i < hashsize / sizeof(u32); ++i)534 atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]);535 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);536 valmr |= SHA_MR_UIHV;537 }538 /*539 * WARNING: If the UIHV feature is not available, the hardware CANNOT540 * process concurrent requests: the internal registers used to store541 * the hash/digest are still set to the partial digest output values542 * computed during the latest round.543 */544 545 atmel_sha_write(dd, SHA_MR, valmr);546}547 548static inline int atmel_sha_wait_for_data_ready(struct atmel_sha_dev *dd,549 atmel_sha_fn_t resume)550{551 u32 isr = atmel_sha_read(dd, SHA_ISR);552 553 if (unlikely(isr & SHA_INT_DATARDY))554 return resume(dd);555 556 dd->resume = resume;557 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);558 return -EINPROGRESS;559}560 561static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,562 size_t length, int final)563{564 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);565 int count, len32;566 const u32 *buffer = (const u32 *)buf;567 568 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",569 ctx->digcnt[1], ctx->digcnt[0], length, final);570 571 atmel_sha_write_ctrl(dd, 0);572 573 /* should be non-zero before next lines to disable clocks later */574 ctx->digcnt[0] += length;575 if (ctx->digcnt[0] < length)576 ctx->digcnt[1]++;577 578 if (final)579 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */580 581 len32 = DIV_ROUND_UP(length, sizeof(u32));582 583 dd->flags |= SHA_FLAGS_CPU;584 585 for (count = 0; count < len32; count++)586 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]);587 588 return -EINPROGRESS;589}590 591static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,592 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)593{594 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);595 int len32;596 597 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",598 ctx->digcnt[1], ctx->digcnt[0], length1, final);599 600 len32 = DIV_ROUND_UP(length1, sizeof(u32));601 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS);602 atmel_sha_write(dd, SHA_TPR, dma_addr1);603 atmel_sha_write(dd, SHA_TCR, len32);604 605 len32 = DIV_ROUND_UP(length2, sizeof(u32));606 atmel_sha_write(dd, SHA_TNPR, dma_addr2);607 atmel_sha_write(dd, SHA_TNCR, len32);608 609 atmel_sha_write_ctrl(dd, 1);610 611 /* should be non-zero before next lines to disable clocks later */612 ctx->digcnt[0] += length1;613 if (ctx->digcnt[0] < length1)614 ctx->digcnt[1]++;615 616 if (final)617 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */618 619 dd->flags |= SHA_FLAGS_DMA_ACTIVE;620 621 /* Start DMA transfer */622 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN);623 624 return -EINPROGRESS;625}626 627static void atmel_sha_dma_callback(void *data)628{629 struct atmel_sha_dev *dd = data;630 631 dd->is_async = true;632 633 /* dma_lch_in - completed - wait DATRDY */634 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);635}636 637static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,638 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)639{640 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);641 struct dma_async_tx_descriptor *in_desc;642 struct scatterlist sg[2];643 644 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",645 ctx->digcnt[1], ctx->digcnt[0], length1, final);646 647 dd->dma_lch_in.dma_conf.src_maxburst = 16;648 dd->dma_lch_in.dma_conf.dst_maxburst = 16;649 650 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);651 652 if (length2) {653 sg_init_table(sg, 2);654 sg_dma_address(&sg[0]) = dma_addr1;655 sg_dma_len(&sg[0]) = length1;656 sg_dma_address(&sg[1]) = dma_addr2;657 sg_dma_len(&sg[1]) = length2;658 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2,659 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);660 } else {661 sg_init_table(sg, 1);662 sg_dma_address(&sg[0]) = dma_addr1;663 sg_dma_len(&sg[0]) = length1;664 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1,665 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);666 }667 if (!in_desc)668 return atmel_sha_complete(dd, -EINVAL);669 670 in_desc->callback = atmel_sha_dma_callback;671 in_desc->callback_param = dd;672 673 atmel_sha_write_ctrl(dd, 1);674 675 /* should be non-zero before next lines to disable clocks later */676 ctx->digcnt[0] += length1;677 if (ctx->digcnt[0] < length1)678 ctx->digcnt[1]++;679 680 if (final)681 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */682 683 dd->flags |= SHA_FLAGS_DMA_ACTIVE;684 685 /* Start DMA transfer */686 dmaengine_submit(in_desc);687 dma_async_issue_pending(dd->dma_lch_in.chan);688 689 return -EINPROGRESS;690}691 692static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,693 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)694{695 if (dd->caps.has_dma)696 return atmel_sha_xmit_dma(dd, dma_addr1, length1,697 dma_addr2, length2, final);698 else699 return atmel_sha_xmit_pdc(dd, dma_addr1, length1,700 dma_addr2, length2, final);701}702 703static int atmel_sha_update_cpu(struct atmel_sha_dev *dd)704{705 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);706 int bufcnt;707 708 atmel_sha_append_sg(ctx);709 atmel_sha_fill_padding(ctx, 0);710 bufcnt = ctx->bufcnt;711 ctx->bufcnt = 0;712 713 return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1);714}715 716static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,717 struct atmel_sha_reqctx *ctx,718 size_t length, int final)719{720 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,721 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);722 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {723 dev_err(dd->dev, "dma %zu bytes error\n", ctx->buflen +724 ctx->block_size);725 return atmel_sha_complete(dd, -EINVAL);726 }727 728 ctx->flags &= ~SHA_FLAGS_SG;729 730 /* next call does not fail... so no unmap in the case of error */731 return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final);732}733 734static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)735{736 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);737 unsigned int final;738 size_t count;739 740 atmel_sha_append_sg(ctx);741 742 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;743 744 dev_dbg(dd->dev, "slow: bufcnt: %zu, digcnt: 0x%llx 0x%llx, final: %d\n",745 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final);746 747 if (final)748 atmel_sha_fill_padding(ctx, 0);749 750 if (final || (ctx->bufcnt == ctx->buflen)) {751 count = ctx->bufcnt;752 ctx->bufcnt = 0;753 return atmel_sha_xmit_dma_map(dd, ctx, count, final);754 }755 756 return 0;757}758 759static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)760{761 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);762 unsigned int length, final, tail;763 struct scatterlist *sg;764 unsigned int count;765 766 if (!ctx->total)767 return 0;768 769 if (ctx->bufcnt || ctx->offset)770 return atmel_sha_update_dma_slow(dd);771 772 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %zd, total: %u\n",773 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total);774 775 sg = ctx->sg;776 777 if (!IS_ALIGNED(sg->offset, sizeof(u32)))778 return atmel_sha_update_dma_slow(dd);779 780 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size))781 /* size is not ctx->block_size aligned */782 return atmel_sha_update_dma_slow(dd);783 784 length = min(ctx->total, sg->length);785 786 if (sg_is_last(sg)) {787 if (!(ctx->flags & SHA_FLAGS_FINUP)) {788 /* not last sg must be ctx->block_size aligned */789 tail = length & (ctx->block_size - 1);790 length -= tail;791 }792 }793 794 ctx->total -= length;795 ctx->offset = length; /* offset where to start slow */796 797 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;798 799 /* Add padding */800 if (final) {801 tail = length & (ctx->block_size - 1);802 length -= tail;803 ctx->total += tail;804 ctx->offset = length; /* offset where to start slow */805 806 sg = ctx->sg;807 atmel_sha_append_sg(ctx);808 809 atmel_sha_fill_padding(ctx, length);810 811 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,812 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);813 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {814 dev_err(dd->dev, "dma %zu bytes error\n",815 ctx->buflen + ctx->block_size);816 return atmel_sha_complete(dd, -EINVAL);817 }818 819 if (length == 0) {820 ctx->flags &= ~SHA_FLAGS_SG;821 count = ctx->bufcnt;822 ctx->bufcnt = 0;823 return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0,824 0, final);825 } else {826 ctx->sg = sg;827 if (!dma_map_sg(dd->dev, ctx->sg, 1,828 DMA_TO_DEVICE)) {829 dev_err(dd->dev, "dma_map_sg error\n");830 return atmel_sha_complete(dd, -EINVAL);831 }832 833 ctx->flags |= SHA_FLAGS_SG;834 835 count = ctx->bufcnt;836 ctx->bufcnt = 0;837 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg),838 length, ctx->dma_addr, count, final);839 }840 }841 842 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {843 dev_err(dd->dev, "dma_map_sg error\n");844 return atmel_sha_complete(dd, -EINVAL);845 }846 847 ctx->flags |= SHA_FLAGS_SG;848 849 /* next call does not fail... so no unmap in the case of error */850 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0,851 0, final);852}853 854static void atmel_sha_update_dma_stop(struct atmel_sha_dev *dd)855{856 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);857 858 if (ctx->flags & SHA_FLAGS_SG) {859 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);860 if (ctx->sg->length == ctx->offset) {861 ctx->sg = sg_next(ctx->sg);862 if (ctx->sg)863 ctx->offset = 0;864 }865 if (ctx->flags & SHA_FLAGS_PAD) {866 dma_unmap_single(dd->dev, ctx->dma_addr,867 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);868 }869 } else {870 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen +871 ctx->block_size, DMA_TO_DEVICE);872 }873}874 875static int atmel_sha_update_req(struct atmel_sha_dev *dd)876{877 struct ahash_request *req = dd->req;878 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);879 int err;880 881 dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n",882 ctx->total, ctx->digcnt[1], ctx->digcnt[0]);883 884 if (ctx->flags & SHA_FLAGS_CPU)885 err = atmel_sha_update_cpu(dd);886 else887 err = atmel_sha_update_dma_start(dd);888 889 /* wait for dma completion before can take more data */890 dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n",891 err, ctx->digcnt[1], ctx->digcnt[0]);892 893 return err;894}895 896static int atmel_sha_final_req(struct atmel_sha_dev *dd)897{898 struct ahash_request *req = dd->req;899 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);900 int err = 0;901 int count;902 903 if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) {904 atmel_sha_fill_padding(ctx, 0);905 count = ctx->bufcnt;906 ctx->bufcnt = 0;907 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1);908 }909 /* faster to handle last block with cpu */910 else {911 atmel_sha_fill_padding(ctx, 0);912 count = ctx->bufcnt;913 ctx->bufcnt = 0;914 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1);915 }916 917 dev_dbg(dd->dev, "final_req: err: %d\n", err);918 919 return err;920}921 922static void atmel_sha_copy_hash(struct ahash_request *req)923{924 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);925 u32 *hash = (u32 *)ctx->digest;926 unsigned int i, hashsize;927 928 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {929 case SHA_FLAGS_SHA1:930 hashsize = SHA1_DIGEST_SIZE;931 break;932 933 case SHA_FLAGS_SHA224:934 case SHA_FLAGS_SHA256:935 hashsize = SHA256_DIGEST_SIZE;936 break;937 938 case SHA_FLAGS_SHA384:939 case SHA_FLAGS_SHA512:940 hashsize = SHA512_DIGEST_SIZE;941 break;942 943 default:944 /* Should not happen... */945 return;946 }947 948 for (i = 0; i < hashsize / sizeof(u32); ++i)949 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));950 ctx->flags |= SHA_FLAGS_RESTORE;951}952 953static void atmel_sha_copy_ready_hash(struct ahash_request *req)954{955 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);956 957 if (!req->result)958 return;959 960 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {961 default:962 case SHA_FLAGS_SHA1:963 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);964 break;965 966 case SHA_FLAGS_SHA224:967 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);968 break;969 970 case SHA_FLAGS_SHA256:971 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);972 break;973 974 case SHA_FLAGS_SHA384:975 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE);976 break;977 978 case SHA_FLAGS_SHA512:979 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE);980 break;981 }982}983 984static int atmel_sha_finish(struct ahash_request *req)985{986 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);987 struct atmel_sha_dev *dd = ctx->dd;988 989 if (ctx->digcnt[0] || ctx->digcnt[1])990 atmel_sha_copy_ready_hash(req);991 992 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %zd\n", ctx->digcnt[1],993 ctx->digcnt[0], ctx->bufcnt);994 995 return 0;996}997 998static void atmel_sha_finish_req(struct ahash_request *req, int err)999{1000 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1001 struct atmel_sha_dev *dd = ctx->dd;1002 1003 if (!err) {1004 atmel_sha_copy_hash(req);1005 if (SHA_FLAGS_FINAL & dd->flags)1006 err = atmel_sha_finish(req);1007 } else {1008 ctx->flags |= SHA_FLAGS_ERROR;1009 }1010 1011 /* atomic operation is not needed here */1012 (void)atmel_sha_complete(dd, err);1013}1014 1015static int atmel_sha_hw_init(struct atmel_sha_dev *dd)1016{1017 int err;1018 1019 err = clk_enable(dd->iclk);1020 if (err)1021 return err;1022 1023 if (!(SHA_FLAGS_INIT & dd->flags)) {1024 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST);1025 dd->flags |= SHA_FLAGS_INIT;1026 }1027 1028 return 0;1029}1030 1031static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd)1032{1033 return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff;1034}1035 1036static int atmel_sha_hw_version_init(struct atmel_sha_dev *dd)1037{1038 int err;1039 1040 err = atmel_sha_hw_init(dd);1041 if (err)1042 return err;1043 1044 dd->hw_version = atmel_sha_get_version(dd);1045 1046 dev_info(dd->dev,1047 "version: 0x%x\n", dd->hw_version);1048 1049 clk_disable(dd->iclk);1050 1051 return 0;1052}1053 1054static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,1055 struct ahash_request *req)1056{1057 struct crypto_async_request *async_req, *backlog;1058 struct atmel_sha_ctx *ctx;1059 unsigned long flags;1060 bool start_async;1061 int err = 0, ret = 0;1062 1063 spin_lock_irqsave(&dd->lock, flags);1064 if (req)1065 ret = ahash_enqueue_request(&dd->queue, req);1066 1067 if (SHA_FLAGS_BUSY & dd->flags) {1068 spin_unlock_irqrestore(&dd->lock, flags);1069 return ret;1070 }1071 1072 backlog = crypto_get_backlog(&dd->queue);1073 async_req = crypto_dequeue_request(&dd->queue);1074 if (async_req)1075 dd->flags |= SHA_FLAGS_BUSY;1076 1077 spin_unlock_irqrestore(&dd->lock, flags);1078 1079 if (!async_req)1080 return ret;1081 1082 if (backlog)1083 crypto_request_complete(backlog, -EINPROGRESS);1084 1085 ctx = crypto_tfm_ctx(async_req->tfm);1086 1087 dd->req = ahash_request_cast(async_req);1088 start_async = (dd->req != req);1089 dd->is_async = start_async;1090 dd->force_complete = false;1091 1092 /* WARNING: ctx->start() MAY change dd->is_async. */1093 err = ctx->start(dd);1094 return (start_async) ? ret : err;1095}1096 1097static int atmel_sha_done(struct atmel_sha_dev *dd);1098 1099static int atmel_sha_start(struct atmel_sha_dev *dd)1100{1101 struct ahash_request *req = dd->req;1102 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1103 int err;1104 1105 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %u\n",1106 ctx->op, req->nbytes);1107 1108 err = atmel_sha_hw_init(dd);1109 if (err)1110 return atmel_sha_complete(dd, err);1111 1112 /*1113 * atmel_sha_update_req() and atmel_sha_final_req() can return either:1114 * -EINPROGRESS: the hardware is busy and the SHA driver will resume1115 * its job later in the done_task.1116 * This is the main path.1117 *1118 * 0: the SHA driver can continue its job then release the hardware1119 * later, if needed, with atmel_sha_finish_req().1120 * This is the alternate path.1121 *1122 * < 0: an error has occurred so atmel_sha_complete(dd, err) has already1123 * been called, hence the hardware has been released.1124 * The SHA driver must stop its job without calling1125 * atmel_sha_finish_req(), otherwise atmel_sha_complete() would be1126 * called a second time.1127 *1128 * Please note that currently, atmel_sha_final_req() never returns 0.1129 */1130 1131 dd->resume = atmel_sha_done;1132 if (ctx->op == SHA_OP_UPDATE) {1133 err = atmel_sha_update_req(dd);1134 if (!err && (ctx->flags & SHA_FLAGS_FINUP))1135 /* no final() after finup() */1136 err = atmel_sha_final_req(dd);1137 } else if (ctx->op == SHA_OP_FINAL) {1138 err = atmel_sha_final_req(dd);1139 }1140 1141 if (!err)1142 /* done_task will not finish it, so do it here */1143 atmel_sha_finish_req(req, err);1144 1145 dev_dbg(dd->dev, "exit, err: %d\n", err);1146 1147 return err;1148}1149 1150static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op)1151{1152 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1153 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);1154 struct atmel_sha_dev *dd = tctx->dd;1155 1156 ctx->op = op;1157 1158 return atmel_sha_handle_queue(dd, req);1159}1160 1161static int atmel_sha_update(struct ahash_request *req)1162{1163 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1164 1165 if (!req->nbytes)1166 return 0;1167 1168 ctx->total = req->nbytes;1169 ctx->sg = req->src;1170 ctx->offset = 0;1171 1172 if (ctx->flags & SHA_FLAGS_FINUP) {1173 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD)1174 /* faster to use CPU for short transfers */1175 ctx->flags |= SHA_FLAGS_CPU;1176 } else if (ctx->bufcnt + ctx->total < ctx->buflen) {1177 atmel_sha_append_sg(ctx);1178 return 0;1179 }1180 return atmel_sha_enqueue(req, SHA_OP_UPDATE);1181}1182 1183static int atmel_sha_final(struct ahash_request *req)1184{1185 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1186 1187 ctx->flags |= SHA_FLAGS_FINUP;1188 1189 if (ctx->flags & SHA_FLAGS_ERROR)1190 return 0; /* uncompleted hash is not needed */1191 1192 if (ctx->flags & SHA_FLAGS_PAD)1193 /* copy ready hash (+ finalize hmac) */1194 return atmel_sha_finish(req);1195 1196 return atmel_sha_enqueue(req, SHA_OP_FINAL);1197}1198 1199static int atmel_sha_finup(struct ahash_request *req)1200{1201 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1202 int err1, err2;1203 1204 ctx->flags |= SHA_FLAGS_FINUP;1205 1206 err1 = atmel_sha_update(req);1207 if (err1 == -EINPROGRESS ||1208 (err1 == -EBUSY && (ahash_request_flags(req) &1209 CRYPTO_TFM_REQ_MAY_BACKLOG)))1210 return err1;1211 1212 /*1213 * final() has to be always called to cleanup resources1214 * even if udpate() failed, except EINPROGRESS1215 */1216 err2 = atmel_sha_final(req);1217 1218 return err1 ?: err2;1219}1220 1221static int atmel_sha_digest(struct ahash_request *req)1222{1223 return atmel_sha_init(req) ?: atmel_sha_finup(req);1224}1225 1226 1227static int atmel_sha_export(struct ahash_request *req, void *out)1228{1229 const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1230 1231 memcpy(out, ctx, sizeof(*ctx));1232 return 0;1233}1234 1235static int atmel_sha_import(struct ahash_request *req, const void *in)1236{1237 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1238 1239 memcpy(ctx, in, sizeof(*ctx));1240 return 0;1241}1242 1243static int atmel_sha_cra_init(struct crypto_tfm *tfm)1244{1245 struct atmel_sha_ctx *ctx = crypto_tfm_ctx(tfm);1246 1247 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),1248 sizeof(struct atmel_sha_reqctx));1249 ctx->start = atmel_sha_start;1250 1251 return 0;1252}1253 1254static void atmel_sha_alg_init(struct ahash_alg *alg)1255{1256 alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY;1257 alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC;1258 alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_ctx);1259 alg->halg.base.cra_module = THIS_MODULE;1260 alg->halg.base.cra_init = atmel_sha_cra_init;1261 1262 alg->halg.statesize = sizeof(struct atmel_sha_reqctx);1263 1264 alg->init = atmel_sha_init;1265 alg->update = atmel_sha_update;1266 alg->final = atmel_sha_final;1267 alg->finup = atmel_sha_finup;1268 alg->digest = atmel_sha_digest;1269 alg->export = atmel_sha_export;1270 alg->import = atmel_sha_import;1271}1272 1273static struct ahash_alg sha_1_256_algs[] = {1274{1275 .halg.base.cra_name = "sha1",1276 .halg.base.cra_driver_name = "atmel-sha1",1277 .halg.base.cra_blocksize = SHA1_BLOCK_SIZE,1278 1279 .halg.digestsize = SHA1_DIGEST_SIZE,1280},1281{1282 .halg.base.cra_name = "sha256",1283 .halg.base.cra_driver_name = "atmel-sha256",1284 .halg.base.cra_blocksize = SHA256_BLOCK_SIZE,1285 1286 .halg.digestsize = SHA256_DIGEST_SIZE,1287},1288};1289 1290static struct ahash_alg sha_224_alg = {1291 .halg.base.cra_name = "sha224",1292 .halg.base.cra_driver_name = "atmel-sha224",1293 .halg.base.cra_blocksize = SHA224_BLOCK_SIZE,1294 1295 .halg.digestsize = SHA224_DIGEST_SIZE,1296};1297 1298static struct ahash_alg sha_384_512_algs[] = {1299{1300 .halg.base.cra_name = "sha384",1301 .halg.base.cra_driver_name = "atmel-sha384",1302 .halg.base.cra_blocksize = SHA384_BLOCK_SIZE,1303 1304 .halg.digestsize = SHA384_DIGEST_SIZE,1305},1306{1307 .halg.base.cra_name = "sha512",1308 .halg.base.cra_driver_name = "atmel-sha512",1309 .halg.base.cra_blocksize = SHA512_BLOCK_SIZE,1310 1311 .halg.digestsize = SHA512_DIGEST_SIZE,1312},1313};1314 1315static void atmel_sha_queue_task(unsigned long data)1316{1317 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;1318 1319 atmel_sha_handle_queue(dd, NULL);1320}1321 1322static int atmel_sha_done(struct atmel_sha_dev *dd)1323{1324 int err = 0;1325 1326 if (SHA_FLAGS_CPU & dd->flags) {1327 if (SHA_FLAGS_OUTPUT_READY & dd->flags) {1328 dd->flags &= ~SHA_FLAGS_OUTPUT_READY;1329 goto finish;1330 }1331 } else if (SHA_FLAGS_DMA_READY & dd->flags) {1332 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) {1333 dd->flags &= ~SHA_FLAGS_DMA_ACTIVE;1334 atmel_sha_update_dma_stop(dd);1335 }1336 if (SHA_FLAGS_OUTPUT_READY & dd->flags) {1337 /* hash or semi-hash ready */1338 dd->flags &= ~(SHA_FLAGS_DMA_READY |1339 SHA_FLAGS_OUTPUT_READY);1340 err = atmel_sha_update_dma_start(dd);1341 if (err != -EINPROGRESS)1342 goto finish;1343 }1344 }1345 return err;1346 1347finish:1348 /* finish curent request */1349 atmel_sha_finish_req(dd->req, err);1350 1351 return err;1352}1353 1354static void atmel_sha_done_task(unsigned long data)1355{1356 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;1357 1358 dd->is_async = true;1359 (void)dd->resume(dd);1360}1361 1362static irqreturn_t atmel_sha_irq(int irq, void *dev_id)1363{1364 struct atmel_sha_dev *sha_dd = dev_id;1365 u32 reg;1366 1367 reg = atmel_sha_read(sha_dd, SHA_ISR);1368 if (reg & atmel_sha_read(sha_dd, SHA_IMR)) {1369 atmel_sha_write(sha_dd, SHA_IDR, reg);1370 if (SHA_FLAGS_BUSY & sha_dd->flags) {1371 sha_dd->flags |= SHA_FLAGS_OUTPUT_READY;1372 if (!(SHA_FLAGS_CPU & sha_dd->flags))1373 sha_dd->flags |= SHA_FLAGS_DMA_READY;1374 tasklet_schedule(&sha_dd->done_task);1375 } else {1376 dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n");1377 }1378 return IRQ_HANDLED;1379 }1380 1381 return IRQ_NONE;1382}1383 1384 1385/* DMA transfer functions */1386 1387static bool atmel_sha_dma_check_aligned(struct atmel_sha_dev *dd,1388 struct scatterlist *sg,1389 size_t len)1390{1391 struct atmel_sha_dma *dma = &dd->dma_lch_in;1392 struct ahash_request *req = dd->req;1393 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1394 size_t bs = ctx->block_size;1395 int nents;1396 1397 for (nents = 0; sg; sg = sg_next(sg), ++nents) {1398 if (!IS_ALIGNED(sg->offset, sizeof(u32)))1399 return false;1400 1401 /*1402 * This is the last sg, the only one that is allowed to1403 * have an unaligned length.1404 */1405 if (len <= sg->length) {1406 dma->nents = nents + 1;1407 dma->last_sg_length = sg->length;1408 sg->length = ALIGN(len, sizeof(u32));1409 return true;1410 }1411 1412 /* All other sg lengths MUST be aligned to the block size. */1413 if (!IS_ALIGNED(sg->length, bs))1414 return false;1415 1416 len -= sg->length;1417 }1418 1419 return false;1420}1421 1422static void atmel_sha_dma_callback2(void *data)1423{1424 struct atmel_sha_dev *dd = data;1425 struct atmel_sha_dma *dma = &dd->dma_lch_in;1426 struct scatterlist *sg;1427 int nents;1428 1429 dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);1430 1431 sg = dma->sg;1432 for (nents = 0; nents < dma->nents - 1; ++nents)1433 sg = sg_next(sg);1434 sg->length = dma->last_sg_length;1435 1436 dd->is_async = true;1437 (void)atmel_sha_wait_for_data_ready(dd, dd->resume);1438}1439 1440static int atmel_sha_dma_start(struct atmel_sha_dev *dd,1441 struct scatterlist *src,1442 size_t len,1443 atmel_sha_fn_t resume)1444{1445 struct atmel_sha_dma *dma = &dd->dma_lch_in;1446 struct dma_slave_config *config = &dma->dma_conf;1447 struct dma_chan *chan = dma->chan;1448 struct dma_async_tx_descriptor *desc;1449 dma_cookie_t cookie;1450 unsigned int sg_len;1451 int err;1452 1453 dd->resume = resume;1454 1455 /*1456 * dma->nents has already been initialized by1457 * atmel_sha_dma_check_aligned().1458 */1459 dma->sg = src;1460 sg_len = dma_map_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);1461 if (!sg_len) {1462 err = -ENOMEM;1463 goto exit;1464 }1465 1466 config->src_maxburst = 16;1467 config->dst_maxburst = 16;1468 err = dmaengine_slave_config(chan, config);1469 if (err)1470 goto unmap_sg;1471 1472 desc = dmaengine_prep_slave_sg(chan, dma->sg, sg_len, DMA_MEM_TO_DEV,1473 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);1474 if (!desc) {1475 err = -ENOMEM;1476 goto unmap_sg;1477 }1478 1479 desc->callback = atmel_sha_dma_callback2;1480 desc->callback_param = dd;1481 cookie = dmaengine_submit(desc);1482 err = dma_submit_error(cookie);1483 if (err)1484 goto unmap_sg;1485 1486 dma_async_issue_pending(chan);1487 1488 return -EINPROGRESS;1489 1490unmap_sg:1491 dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE);1492exit:1493 return atmel_sha_complete(dd, err);1494}1495 1496 1497/* CPU transfer functions */1498 1499static int atmel_sha_cpu_transfer(struct atmel_sha_dev *dd)1500{1501 struct ahash_request *req = dd->req;1502 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1503 const u32 *words = (const u32 *)ctx->buffer;1504 size_t i, num_words;1505 u32 isr, din, din_inc;1506 1507 din_inc = (ctx->flags & SHA_FLAGS_IDATAR0) ? 0 : 1;1508 for (;;) {1509 /* Write data into the Input Data Registers. */1510 num_words = DIV_ROUND_UP(ctx->bufcnt, sizeof(u32));1511 for (i = 0, din = 0; i < num_words; ++i, din += din_inc)1512 atmel_sha_write(dd, SHA_REG_DIN(din), words[i]);1513 1514 ctx->offset += ctx->bufcnt;1515 ctx->total -= ctx->bufcnt;1516 1517 if (!ctx->total)1518 break;1519 1520 /*1521 * Prepare next block:1522 * Fill ctx->buffer now with the next data to be written into1523 * IDATARx: it gives time for the SHA hardware to process1524 * the current data so the SHA_INT_DATARDY flag might be set1525 * in SHA_ISR when polling this register at the beginning of1526 * the next loop.1527 */1528 ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total);1529 scatterwalk_map_and_copy(ctx->buffer, ctx->sg,1530 ctx->offset, ctx->bufcnt, 0);1531 1532 /* Wait for hardware to be ready again. */1533 isr = atmel_sha_read(dd, SHA_ISR);1534 if (!(isr & SHA_INT_DATARDY)) {1535 /* Not ready yet. */1536 dd->resume = atmel_sha_cpu_transfer;1537 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);1538 return -EINPROGRESS;1539 }1540 }1541 1542 if (unlikely(!(ctx->flags & SHA_FLAGS_WAIT_DATARDY)))1543 return dd->cpu_transfer_complete(dd);1544 1545 return atmel_sha_wait_for_data_ready(dd, dd->cpu_transfer_complete);1546}1547 1548static int atmel_sha_cpu_start(struct atmel_sha_dev *dd,1549 struct scatterlist *sg,1550 unsigned int len,1551 bool idatar0_only,1552 bool wait_data_ready,1553 atmel_sha_fn_t resume)1554{1555 struct ahash_request *req = dd->req;1556 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1557 1558 if (!len)1559 return resume(dd);1560 1561 ctx->flags &= ~(SHA_FLAGS_IDATAR0 | SHA_FLAGS_WAIT_DATARDY);1562 1563 if (idatar0_only)1564 ctx->flags |= SHA_FLAGS_IDATAR0;1565 1566 if (wait_data_ready)1567 ctx->flags |= SHA_FLAGS_WAIT_DATARDY;1568 1569 ctx->sg = sg;1570 ctx->total = len;1571 ctx->offset = 0;1572 1573 /* Prepare the first block to be written. */1574 ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total);1575 scatterwalk_map_and_copy(ctx->buffer, ctx->sg,1576 ctx->offset, ctx->bufcnt, 0);1577 1578 dd->cpu_transfer_complete = resume;1579 return atmel_sha_cpu_transfer(dd);1580}1581 1582static int atmel_sha_cpu_hash(struct atmel_sha_dev *dd,1583 const void *data, unsigned int datalen,1584 bool auto_padding,1585 atmel_sha_fn_t resume)1586{1587 struct ahash_request *req = dd->req;1588 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1589 u32 msglen = (auto_padding) ? datalen : 0;1590 u32 mr = SHA_MR_MODE_AUTO;1591 1592 if (!(IS_ALIGNED(datalen, ctx->block_size) || auto_padding))1593 return atmel_sha_complete(dd, -EINVAL);1594 1595 mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK);1596 atmel_sha_write(dd, SHA_MR, mr);1597 atmel_sha_write(dd, SHA_MSR, msglen);1598 atmel_sha_write(dd, SHA_BCR, msglen);1599 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);1600 1601 sg_init_one(&dd->tmp, data, datalen);1602 return atmel_sha_cpu_start(dd, &dd->tmp, datalen, false, true, resume);1603}1604 1605 1606/* hmac functions */1607 1608struct atmel_sha_hmac_key {1609 bool valid;1610 unsigned int keylen;1611 u8 buffer[SHA512_BLOCK_SIZE];1612 u8 *keydup;1613};1614 1615static inline void atmel_sha_hmac_key_init(struct atmel_sha_hmac_key *hkey)1616{1617 memset(hkey, 0, sizeof(*hkey));1618}1619 1620static inline void atmel_sha_hmac_key_release(struct atmel_sha_hmac_key *hkey)1621{1622 kfree(hkey->keydup);1623 memset(hkey, 0, sizeof(*hkey));1624}1625 1626static inline int atmel_sha_hmac_key_set(struct atmel_sha_hmac_key *hkey,1627 const u8 *key,1628 unsigned int keylen)1629{1630 atmel_sha_hmac_key_release(hkey);1631 1632 if (keylen > sizeof(hkey->buffer)) {1633 hkey->keydup = kmemdup(key, keylen, GFP_KERNEL);1634 if (!hkey->keydup)1635 return -ENOMEM;1636 1637 } else {1638 memcpy(hkey->buffer, key, keylen);1639 }1640 1641 hkey->valid = true;1642 hkey->keylen = keylen;1643 return 0;1644}1645 1646static inline bool atmel_sha_hmac_key_get(const struct atmel_sha_hmac_key *hkey,1647 const u8 **key,1648 unsigned int *keylen)1649{1650 if (!hkey->valid)1651 return false;1652 1653 *keylen = hkey->keylen;1654 *key = (hkey->keydup) ? hkey->keydup : hkey->buffer;1655 return true;1656}1657 1658 1659struct atmel_sha_hmac_ctx {1660 struct atmel_sha_ctx base;1661 1662 struct atmel_sha_hmac_key hkey;1663 u32 ipad[SHA512_BLOCK_SIZE / sizeof(u32)];1664 u32 opad[SHA512_BLOCK_SIZE / sizeof(u32)];1665 atmel_sha_fn_t resume;1666};1667 1668static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd,1669 atmel_sha_fn_t resume);1670static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd,1671 const u8 *key, unsigned int keylen);1672static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd);1673static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd);1674static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd);1675static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd);1676 1677static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd);1678static int atmel_sha_hmac_final(struct atmel_sha_dev *dd);1679static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd);1680static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd);1681 1682static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd,1683 atmel_sha_fn_t resume)1684{1685 struct ahash_request *req = dd->req;1686 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1687 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1688 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1689 unsigned int keylen;1690 const u8 *key;1691 size_t bs;1692 1693 hmac->resume = resume;1694 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {1695 case SHA_FLAGS_SHA1:1696 ctx->block_size = SHA1_BLOCK_SIZE;1697 ctx->hash_size = SHA1_DIGEST_SIZE;1698 break;1699 1700 case SHA_FLAGS_SHA224:1701 ctx->block_size = SHA224_BLOCK_SIZE;1702 ctx->hash_size = SHA256_DIGEST_SIZE;1703 break;1704 1705 case SHA_FLAGS_SHA256:1706 ctx->block_size = SHA256_BLOCK_SIZE;1707 ctx->hash_size = SHA256_DIGEST_SIZE;1708 break;1709 1710 case SHA_FLAGS_SHA384:1711 ctx->block_size = SHA384_BLOCK_SIZE;1712 ctx->hash_size = SHA512_DIGEST_SIZE;1713 break;1714 1715 case SHA_FLAGS_SHA512:1716 ctx->block_size = SHA512_BLOCK_SIZE;1717 ctx->hash_size = SHA512_DIGEST_SIZE;1718 break;1719 1720 default:1721 return atmel_sha_complete(dd, -EINVAL);1722 }1723 bs = ctx->block_size;1724 1725 if (likely(!atmel_sha_hmac_key_get(&hmac->hkey, &key, &keylen)))1726 return resume(dd);1727 1728 /* Compute K' from K. */1729 if (unlikely(keylen > bs))1730 return atmel_sha_hmac_prehash_key(dd, key, keylen);1731 1732 /* Prepare ipad. */1733 memcpy((u8 *)hmac->ipad, key, keylen);1734 memset((u8 *)hmac->ipad + keylen, 0, bs - keylen);1735 return atmel_sha_hmac_compute_ipad_hash(dd);1736}1737 1738static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd,1739 const u8 *key, unsigned int keylen)1740{1741 return atmel_sha_cpu_hash(dd, key, keylen, true,1742 atmel_sha_hmac_prehash_key_done);1743}1744 1745static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd)1746{1747 struct ahash_request *req = dd->req;1748 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1749 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1750 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1751 size_t ds = crypto_ahash_digestsize(tfm);1752 size_t bs = ctx->block_size;1753 size_t i, num_words = ds / sizeof(u32);1754 1755 /* Prepare ipad. */1756 for (i = 0; i < num_words; ++i)1757 hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));1758 memset((u8 *)hmac->ipad + ds, 0, bs - ds);1759 return atmel_sha_hmac_compute_ipad_hash(dd);1760}1761 1762static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd)1763{1764 struct ahash_request *req = dd->req;1765 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1766 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1767 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1768 size_t bs = ctx->block_size;1769 size_t i, num_words = bs / sizeof(u32);1770 1771 unsafe_memcpy(hmac->opad, hmac->ipad, bs,1772 "fortified memcpy causes -Wrestrict warning");1773 for (i = 0; i < num_words; ++i) {1774 hmac->ipad[i] ^= 0x36363636;1775 hmac->opad[i] ^= 0x5c5c5c5c;1776 }1777 1778 return atmel_sha_cpu_hash(dd, hmac->ipad, bs, false,1779 atmel_sha_hmac_compute_opad_hash);1780}1781 1782static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd)1783{1784 struct ahash_request *req = dd->req;1785 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1786 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1787 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1788 size_t bs = ctx->block_size;1789 size_t hs = ctx->hash_size;1790 size_t i, num_words = hs / sizeof(u32);1791 1792 for (i = 0; i < num_words; ++i)1793 hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));1794 return atmel_sha_cpu_hash(dd, hmac->opad, bs, false,1795 atmel_sha_hmac_setup_done);1796}1797 1798static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd)1799{1800 struct ahash_request *req = dd->req;1801 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1802 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1803 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1804 size_t hs = ctx->hash_size;1805 size_t i, num_words = hs / sizeof(u32);1806 1807 for (i = 0; i < num_words; ++i)1808 hmac->opad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));1809 atmel_sha_hmac_key_release(&hmac->hkey);1810 return hmac->resume(dd);1811}1812 1813static int atmel_sha_hmac_start(struct atmel_sha_dev *dd)1814{1815 struct ahash_request *req = dd->req;1816 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1817 int err;1818 1819 err = atmel_sha_hw_init(dd);1820 if (err)1821 return atmel_sha_complete(dd, err);1822 1823 switch (ctx->op) {1824 case SHA_OP_INIT:1825 err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_init_done);1826 break;1827 1828 case SHA_OP_UPDATE:1829 dd->resume = atmel_sha_done;1830 err = atmel_sha_update_req(dd);1831 break;1832 1833 case SHA_OP_FINAL:1834 dd->resume = atmel_sha_hmac_final;1835 err = atmel_sha_final_req(dd);1836 break;1837 1838 case SHA_OP_DIGEST:1839 err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_digest2);1840 break;1841 1842 default:1843 return atmel_sha_complete(dd, -EINVAL);1844 }1845 1846 return err;1847}1848 1849static int atmel_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,1850 unsigned int keylen)1851{1852 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1853 1854 return atmel_sha_hmac_key_set(&hmac->hkey, key, keylen);1855}1856 1857static int atmel_sha_hmac_init(struct ahash_request *req)1858{1859 int err;1860 1861 err = atmel_sha_init(req);1862 if (err)1863 return err;1864 1865 return atmel_sha_enqueue(req, SHA_OP_INIT);1866}1867 1868static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd)1869{1870 struct ahash_request *req = dd->req;1871 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1872 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1873 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1874 size_t bs = ctx->block_size;1875 size_t hs = ctx->hash_size;1876 1877 ctx->bufcnt = 0;1878 ctx->digcnt[0] = bs;1879 ctx->digcnt[1] = 0;1880 ctx->flags |= SHA_FLAGS_RESTORE;1881 memcpy(ctx->digest, hmac->ipad, hs);1882 return atmel_sha_complete(dd, 0);1883}1884 1885static int atmel_sha_hmac_final(struct atmel_sha_dev *dd)1886{1887 struct ahash_request *req = dd->req;1888 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1889 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1890 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1891 u32 *digest = (u32 *)ctx->digest;1892 size_t ds = crypto_ahash_digestsize(tfm);1893 size_t bs = ctx->block_size;1894 size_t hs = ctx->hash_size;1895 size_t i, num_words;1896 u32 mr;1897 1898 /* Save d = SHA((K' + ipad) | msg). */1899 num_words = ds / sizeof(u32);1900 for (i = 0; i < num_words; ++i)1901 digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));1902 1903 /* Restore context to finish computing SHA((K' + opad) | d). */1904 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);1905 num_words = hs / sizeof(u32);1906 for (i = 0; i < num_words; ++i)1907 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);1908 1909 mr = SHA_MR_MODE_AUTO | SHA_MR_UIHV;1910 mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK);1911 atmel_sha_write(dd, SHA_MR, mr);1912 atmel_sha_write(dd, SHA_MSR, bs + ds);1913 atmel_sha_write(dd, SHA_BCR, ds);1914 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);1915 1916 sg_init_one(&dd->tmp, digest, ds);1917 return atmel_sha_cpu_start(dd, &dd->tmp, ds, false, true,1918 atmel_sha_hmac_final_done);1919}1920 1921static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd)1922{1923 /*1924 * req->result might not be sizeof(u32) aligned, so copy the1925 * digest into ctx->digest[] before memcpy() the data into1926 * req->result.1927 */1928 atmel_sha_copy_hash(dd->req);1929 atmel_sha_copy_ready_hash(dd->req);1930 return atmel_sha_complete(dd, 0);1931}1932 1933static int atmel_sha_hmac_digest(struct ahash_request *req)1934{1935 int err;1936 1937 err = atmel_sha_init(req);1938 if (err)1939 return err;1940 1941 return atmel_sha_enqueue(req, SHA_OP_DIGEST);1942}1943 1944static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd)1945{1946 struct ahash_request *req = dd->req;1947 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);1948 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1949 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);1950 struct scatterlist *sgbuf;1951 size_t hs = ctx->hash_size;1952 size_t i, num_words = hs / sizeof(u32);1953 bool use_dma = false;1954 u32 mr;1955 1956 /* Special case for empty message. */1957 if (!req->nbytes) {1958 req->nbytes = 0;1959 ctx->bufcnt = 0;1960 ctx->digcnt[0] = 0;1961 ctx->digcnt[1] = 0;1962 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {1963 case SHA_FLAGS_SHA1:1964 case SHA_FLAGS_SHA224:1965 case SHA_FLAGS_SHA256:1966 atmel_sha_fill_padding(ctx, 64);1967 break;1968 1969 case SHA_FLAGS_SHA384:1970 case SHA_FLAGS_SHA512:1971 atmel_sha_fill_padding(ctx, 128);1972 break;1973 }1974 sg_init_one(&dd->tmp, ctx->buffer, ctx->bufcnt);1975 }1976 1977 /* Check DMA threshold and alignment. */1978 if (req->nbytes > ATMEL_SHA_DMA_THRESHOLD &&1979 atmel_sha_dma_check_aligned(dd, req->src, req->nbytes))1980 use_dma = true;1981 1982 /* Write both initial hash values to compute a HMAC. */1983 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);1984 for (i = 0; i < num_words; ++i)1985 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]);1986 1987 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV);1988 for (i = 0; i < num_words; ++i)1989 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);1990 1991 /* Write the Mode, Message Size, Bytes Count then Control Registers. */1992 mr = (SHA_MR_HMAC | SHA_MR_DUALBUFF);1993 mr |= ctx->flags & SHA_FLAGS_ALGO_MASK;1994 if (use_dma)1995 mr |= SHA_MR_MODE_IDATAR0;1996 else1997 mr |= SHA_MR_MODE_AUTO;1998 atmel_sha_write(dd, SHA_MR, mr);1999 2000 atmel_sha_write(dd, SHA_MSR, req->nbytes);2001 atmel_sha_write(dd, SHA_BCR, req->nbytes);2002 2003 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);2004 2005 /* Special case for empty message. */2006 if (!req->nbytes) {2007 sgbuf = &dd->tmp;2008 req->nbytes = ctx->bufcnt;2009 } else {2010 sgbuf = req->src;2011 }2012 2013 /* Process data. */2014 if (use_dma)2015 return atmel_sha_dma_start(dd, sgbuf, req->nbytes,2016 atmel_sha_hmac_final_done);2017 2018 return atmel_sha_cpu_start(dd, sgbuf, req->nbytes, false, true,2019 atmel_sha_hmac_final_done);2020}2021 2022static int atmel_sha_hmac_cra_init(struct crypto_tfm *tfm)2023{2024 struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm);2025 2026 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),2027 sizeof(struct atmel_sha_reqctx));2028 hmac->base.start = atmel_sha_hmac_start;2029 atmel_sha_hmac_key_init(&hmac->hkey);2030 2031 return 0;2032}2033 2034static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm)2035{2036 struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm);2037 2038 atmel_sha_hmac_key_release(&hmac->hkey);2039}2040 2041static void atmel_sha_hmac_alg_init(struct ahash_alg *alg)2042{2043 alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY;2044 alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC;2045 alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx);2046 alg->halg.base.cra_module = THIS_MODULE;2047 alg->halg.base.cra_init = atmel_sha_hmac_cra_init;2048 alg->halg.base.cra_exit = atmel_sha_hmac_cra_exit;2049 2050 alg->halg.statesize = sizeof(struct atmel_sha_reqctx);2051 2052 alg->init = atmel_sha_hmac_init;2053 alg->update = atmel_sha_update;2054 alg->final = atmel_sha_final;2055 alg->digest = atmel_sha_hmac_digest;2056 alg->setkey = atmel_sha_hmac_setkey;2057 alg->export = atmel_sha_export;2058 alg->import = atmel_sha_import;2059}2060 2061static struct ahash_alg sha_hmac_algs[] = {2062{2063 .halg.base.cra_name = "hmac(sha1)",2064 .halg.base.cra_driver_name = "atmel-hmac-sha1",2065 .halg.base.cra_blocksize = SHA1_BLOCK_SIZE,2066 2067 .halg.digestsize = SHA1_DIGEST_SIZE,2068},2069{2070 .halg.base.cra_name = "hmac(sha224)",2071 .halg.base.cra_driver_name = "atmel-hmac-sha224",2072 .halg.base.cra_blocksize = SHA224_BLOCK_SIZE,2073 2074 .halg.digestsize = SHA224_DIGEST_SIZE,2075},2076{2077 .halg.base.cra_name = "hmac(sha256)",2078 .halg.base.cra_driver_name = "atmel-hmac-sha256",2079 .halg.base.cra_blocksize = SHA256_BLOCK_SIZE,2080 2081 .halg.digestsize = SHA256_DIGEST_SIZE,2082},2083{2084 .halg.base.cra_name = "hmac(sha384)",2085 .halg.base.cra_driver_name = "atmel-hmac-sha384",2086 .halg.base.cra_blocksize = SHA384_BLOCK_SIZE,2087 2088 .halg.digestsize = SHA384_DIGEST_SIZE,2089},2090{2091 .halg.base.cra_name = "hmac(sha512)",2092 .halg.base.cra_driver_name = "atmel-hmac-sha512",2093 .halg.base.cra_blocksize = SHA512_BLOCK_SIZE,2094 2095 .halg.digestsize = SHA512_DIGEST_SIZE,2096},2097};2098 2099#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)2100/* authenc functions */2101 2102static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd);2103static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd);2104static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd);2105 2106 2107struct atmel_sha_authenc_ctx {2108 struct crypto_ahash *tfm;2109};2110 2111struct atmel_sha_authenc_reqctx {2112 struct atmel_sha_reqctx base;2113 2114 atmel_aes_authenc_fn_t cb;2115 struct atmel_aes_dev *aes_dev;2116 2117 /* _init() parameters. */2118 struct scatterlist *assoc;2119 u32 assoclen;2120 u32 textlen;2121 2122 /* _final() parameters. */2123 u32 *digest;2124 unsigned int digestlen;2125};2126 2127static void atmel_sha_authenc_complete(void *data, int err)2128{2129 struct ahash_request *req = data;2130 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2131 2132 authctx->cb(authctx->aes_dev, err, authctx->base.dd->is_async);2133}2134 2135static int atmel_sha_authenc_start(struct atmel_sha_dev *dd)2136{2137 struct ahash_request *req = dd->req;2138 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2139 int err;2140 2141 /*2142 * Force atmel_sha_complete() to call req->base.complete(), ie2143 * atmel_sha_authenc_complete(), which in turn calls authctx->cb().2144 */2145 dd->force_complete = true;2146 2147 err = atmel_sha_hw_init(dd);2148 return authctx->cb(authctx->aes_dev, err, dd->is_async);2149}2150 2151bool atmel_sha_authenc_is_ready(void)2152{2153 struct atmel_sha_ctx dummy;2154 2155 dummy.dd = NULL;2156 return (atmel_sha_find_dev(&dummy) != NULL);2157}2158EXPORT_SYMBOL_GPL(atmel_sha_authenc_is_ready);2159 2160unsigned int atmel_sha_authenc_get_reqsize(void)2161{2162 return sizeof(struct atmel_sha_authenc_reqctx);2163}2164EXPORT_SYMBOL_GPL(atmel_sha_authenc_get_reqsize);2165 2166struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode)2167{2168 struct atmel_sha_authenc_ctx *auth;2169 struct crypto_ahash *tfm;2170 struct atmel_sha_ctx *tctx;2171 const char *name;2172 int err = -EINVAL;2173 2174 switch (mode & SHA_FLAGS_MODE_MASK) {2175 case SHA_FLAGS_HMAC_SHA1:2176 name = "atmel-hmac-sha1";2177 break;2178 2179 case SHA_FLAGS_HMAC_SHA224:2180 name = "atmel-hmac-sha224";2181 break;2182 2183 case SHA_FLAGS_HMAC_SHA256:2184 name = "atmel-hmac-sha256";2185 break;2186 2187 case SHA_FLAGS_HMAC_SHA384:2188 name = "atmel-hmac-sha384";2189 break;2190 2191 case SHA_FLAGS_HMAC_SHA512:2192 name = "atmel-hmac-sha512";2193 break;2194 2195 default:2196 goto error;2197 }2198 2199 tfm = crypto_alloc_ahash(name, 0, 0);2200 if (IS_ERR(tfm)) {2201 err = PTR_ERR(tfm);2202 goto error;2203 }2204 tctx = crypto_ahash_ctx(tfm);2205 tctx->start = atmel_sha_authenc_start;2206 tctx->flags = mode;2207 2208 auth = kzalloc(sizeof(*auth), GFP_KERNEL);2209 if (!auth) {2210 err = -ENOMEM;2211 goto err_free_ahash;2212 }2213 auth->tfm = tfm;2214 2215 return auth;2216 2217err_free_ahash:2218 crypto_free_ahash(tfm);2219error:2220 return ERR_PTR(err);2221}2222EXPORT_SYMBOL_GPL(atmel_sha_authenc_spawn);2223 2224void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth)2225{2226 if (auth)2227 crypto_free_ahash(auth->tfm);2228 kfree(auth);2229}2230EXPORT_SYMBOL_GPL(atmel_sha_authenc_free);2231 2232int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth,2233 const u8 *key, unsigned int keylen, u32 flags)2234{2235 struct crypto_ahash *tfm = auth->tfm;2236 2237 crypto_ahash_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);2238 crypto_ahash_set_flags(tfm, flags & CRYPTO_TFM_REQ_MASK);2239 return crypto_ahash_setkey(tfm, key, keylen);2240}2241EXPORT_SYMBOL_GPL(atmel_sha_authenc_setkey);2242 2243int atmel_sha_authenc_schedule(struct ahash_request *req,2244 struct atmel_sha_authenc_ctx *auth,2245 atmel_aes_authenc_fn_t cb,2246 struct atmel_aes_dev *aes_dev)2247{2248 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2249 struct atmel_sha_reqctx *ctx = &authctx->base;2250 struct crypto_ahash *tfm = auth->tfm;2251 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);2252 struct atmel_sha_dev *dd;2253 2254 /* Reset request context (MUST be done first). */2255 memset(authctx, 0, sizeof(*authctx));2256 2257 /* Get SHA device. */2258 dd = atmel_sha_find_dev(tctx);2259 if (!dd)2260 return cb(aes_dev, -ENODEV, false);2261 2262 /* Init request context. */2263 ctx->dd = dd;2264 ctx->buflen = SHA_BUFFER_LEN;2265 authctx->cb = cb;2266 authctx->aes_dev = aes_dev;2267 ahash_request_set_tfm(req, tfm);2268 ahash_request_set_callback(req, 0, atmel_sha_authenc_complete, req);2269 2270 return atmel_sha_handle_queue(dd, req);2271}2272EXPORT_SYMBOL_GPL(atmel_sha_authenc_schedule);2273 2274int atmel_sha_authenc_init(struct ahash_request *req,2275 struct scatterlist *assoc, unsigned int assoclen,2276 unsigned int textlen,2277 atmel_aes_authenc_fn_t cb,2278 struct atmel_aes_dev *aes_dev)2279{2280 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2281 struct atmel_sha_reqctx *ctx = &authctx->base;2282 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);2283 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);2284 struct atmel_sha_dev *dd = ctx->dd;2285 2286 if (unlikely(!IS_ALIGNED(assoclen, sizeof(u32))))2287 return atmel_sha_complete(dd, -EINVAL);2288 2289 authctx->cb = cb;2290 authctx->aes_dev = aes_dev;2291 authctx->assoc = assoc;2292 authctx->assoclen = assoclen;2293 authctx->textlen = textlen;2294 2295 ctx->flags = hmac->base.flags;2296 return atmel_sha_hmac_setup(dd, atmel_sha_authenc_init2);2297}2298EXPORT_SYMBOL_GPL(atmel_sha_authenc_init);2299 2300static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd)2301{2302 struct ahash_request *req = dd->req;2303 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2304 struct atmel_sha_reqctx *ctx = &authctx->base;2305 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);2306 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm);2307 size_t hs = ctx->hash_size;2308 size_t i, num_words = hs / sizeof(u32);2309 u32 mr, msg_size;2310 2311 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);2312 for (i = 0; i < num_words; ++i)2313 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]);2314 2315 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV);2316 for (i = 0; i < num_words; ++i)2317 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]);2318 2319 mr = (SHA_MR_MODE_IDATAR0 |2320 SHA_MR_HMAC |2321 SHA_MR_DUALBUFF);2322 mr |= ctx->flags & SHA_FLAGS_ALGO_MASK;2323 atmel_sha_write(dd, SHA_MR, mr);2324 2325 msg_size = authctx->assoclen + authctx->textlen;2326 atmel_sha_write(dd, SHA_MSR, msg_size);2327 atmel_sha_write(dd, SHA_BCR, msg_size);2328 2329 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);2330 2331 /* Process assoc data. */2332 return atmel_sha_cpu_start(dd, authctx->assoc, authctx->assoclen,2333 true, false,2334 atmel_sha_authenc_init_done);2335}2336 2337static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd)2338{2339 struct ahash_request *req = dd->req;2340 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2341 2342 return authctx->cb(authctx->aes_dev, 0, dd->is_async);2343}2344 2345int atmel_sha_authenc_final(struct ahash_request *req,2346 u32 *digest, unsigned int digestlen,2347 atmel_aes_authenc_fn_t cb,2348 struct atmel_aes_dev *aes_dev)2349{2350 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2351 struct atmel_sha_reqctx *ctx = &authctx->base;2352 struct atmel_sha_dev *dd = ctx->dd;2353 2354 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {2355 case SHA_FLAGS_SHA1:2356 authctx->digestlen = SHA1_DIGEST_SIZE;2357 break;2358 2359 case SHA_FLAGS_SHA224:2360 authctx->digestlen = SHA224_DIGEST_SIZE;2361 break;2362 2363 case SHA_FLAGS_SHA256:2364 authctx->digestlen = SHA256_DIGEST_SIZE;2365 break;2366 2367 case SHA_FLAGS_SHA384:2368 authctx->digestlen = SHA384_DIGEST_SIZE;2369 break;2370 2371 case SHA_FLAGS_SHA512:2372 authctx->digestlen = SHA512_DIGEST_SIZE;2373 break;2374 2375 default:2376 return atmel_sha_complete(dd, -EINVAL);2377 }2378 if (authctx->digestlen > digestlen)2379 authctx->digestlen = digestlen;2380 2381 authctx->cb = cb;2382 authctx->aes_dev = aes_dev;2383 authctx->digest = digest;2384 return atmel_sha_wait_for_data_ready(dd,2385 atmel_sha_authenc_final_done);2386}2387EXPORT_SYMBOL_GPL(atmel_sha_authenc_final);2388 2389static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd)2390{2391 struct ahash_request *req = dd->req;2392 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2393 size_t i, num_words = authctx->digestlen / sizeof(u32);2394 2395 for (i = 0; i < num_words; ++i)2396 authctx->digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i));2397 2398 return atmel_sha_complete(dd, 0);2399}2400 2401void atmel_sha_authenc_abort(struct ahash_request *req)2402{2403 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req);2404 struct atmel_sha_reqctx *ctx = &authctx->base;2405 struct atmel_sha_dev *dd = ctx->dd;2406 2407 /* Prevent atmel_sha_complete() from calling req->base.complete(). */2408 dd->is_async = false;2409 dd->force_complete = false;2410 (void)atmel_sha_complete(dd, 0);2411}2412EXPORT_SYMBOL_GPL(atmel_sha_authenc_abort);2413 2414#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */2415 2416 2417static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)2418{2419 int i;2420 2421 if (dd->caps.has_hmac)2422 for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++)2423 crypto_unregister_ahash(&sha_hmac_algs[i]);2424 2425 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++)2426 crypto_unregister_ahash(&sha_1_256_algs[i]);2427 2428 if (dd->caps.has_sha224)2429 crypto_unregister_ahash(&sha_224_alg);2430 2431 if (dd->caps.has_sha_384_512) {2432 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++)2433 crypto_unregister_ahash(&sha_384_512_algs[i]);2434 }2435}2436 2437static int atmel_sha_register_algs(struct atmel_sha_dev *dd)2438{2439 int err, i, j;2440 2441 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) {2442 atmel_sha_alg_init(&sha_1_256_algs[i]);2443 2444 err = crypto_register_ahash(&sha_1_256_algs[i]);2445 if (err)2446 goto err_sha_1_256_algs;2447 }2448 2449 if (dd->caps.has_sha224) {2450 atmel_sha_alg_init(&sha_224_alg);2451 2452 err = crypto_register_ahash(&sha_224_alg);2453 if (err)2454 goto err_sha_224_algs;2455 }2456 2457 if (dd->caps.has_sha_384_512) {2458 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) {2459 atmel_sha_alg_init(&sha_384_512_algs[i]);2460 2461 err = crypto_register_ahash(&sha_384_512_algs[i]);2462 if (err)2463 goto err_sha_384_512_algs;2464 }2465 }2466 2467 if (dd->caps.has_hmac) {2468 for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) {2469 atmel_sha_hmac_alg_init(&sha_hmac_algs[i]);2470 2471 err = crypto_register_ahash(&sha_hmac_algs[i]);2472 if (err)2473 goto err_sha_hmac_algs;2474 }2475 }2476 2477 return 0;2478 2479 /*i = ARRAY_SIZE(sha_hmac_algs);*/2480err_sha_hmac_algs:2481 for (j = 0; j < i; j++)2482 crypto_unregister_ahash(&sha_hmac_algs[j]);2483 i = ARRAY_SIZE(sha_384_512_algs);2484err_sha_384_512_algs:2485 for (j = 0; j < i; j++)2486 crypto_unregister_ahash(&sha_384_512_algs[j]);2487 crypto_unregister_ahash(&sha_224_alg);2488err_sha_224_algs:2489 i = ARRAY_SIZE(sha_1_256_algs);2490err_sha_1_256_algs:2491 for (j = 0; j < i; j++)2492 crypto_unregister_ahash(&sha_1_256_algs[j]);2493 2494 return err;2495}2496 2497static int atmel_sha_dma_init(struct atmel_sha_dev *dd)2498{2499 dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx");2500 if (IS_ERR(dd->dma_lch_in.chan)) {2501 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_lch_in.chan),2502 "DMA channel is not available\n");2503 }2504 2505 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +2506 SHA_REG_DIN(0);2507 dd->dma_lch_in.dma_conf.src_maxburst = 1;2508 dd->dma_lch_in.dma_conf.src_addr_width =2509 DMA_SLAVE_BUSWIDTH_4_BYTES;2510 dd->dma_lch_in.dma_conf.dst_maxburst = 1;2511 dd->dma_lch_in.dma_conf.dst_addr_width =2512 DMA_SLAVE_BUSWIDTH_4_BYTES;2513 dd->dma_lch_in.dma_conf.device_fc = false;2514 2515 return 0;2516}2517 2518static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd)2519{2520 dma_release_channel(dd->dma_lch_in.chan);2521}2522 2523static void atmel_sha_get_cap(struct atmel_sha_dev *dd)2524{2525 2526 dd->caps.has_dma = 0;2527 dd->caps.has_dualbuff = 0;2528 dd->caps.has_sha224 = 0;2529 dd->caps.has_sha_384_512 = 0;2530 dd->caps.has_uihv = 0;2531 dd->caps.has_hmac = 0;2532 2533 /* keep only major version number */2534 switch (dd->hw_version & 0xff0) {2535 case 0x700:2536 case 0x600:2537 case 0x510:2538 dd->caps.has_dma = 1;2539 dd->caps.has_dualbuff = 1;2540 dd->caps.has_sha224 = 1;2541 dd->caps.has_sha_384_512 = 1;2542 dd->caps.has_uihv = 1;2543 dd->caps.has_hmac = 1;2544 break;2545 case 0x420:2546 dd->caps.has_dma = 1;2547 dd->caps.has_dualbuff = 1;2548 dd->caps.has_sha224 = 1;2549 dd->caps.has_sha_384_512 = 1;2550 dd->caps.has_uihv = 1;2551 break;2552 case 0x410:2553 dd->caps.has_dma = 1;2554 dd->caps.has_dualbuff = 1;2555 dd->caps.has_sha224 = 1;2556 dd->caps.has_sha_384_512 = 1;2557 break;2558 case 0x400:2559 dd->caps.has_dma = 1;2560 dd->caps.has_dualbuff = 1;2561 dd->caps.has_sha224 = 1;2562 break;2563 case 0x320:2564 break;2565 default:2566 dev_warn(dd->dev,2567 "Unmanaged sha version, set minimum capabilities\n");2568 break;2569 }2570}2571 2572static const struct of_device_id atmel_sha_dt_ids[] = {2573 { .compatible = "atmel,at91sam9g46-sha" },2574 { /* sentinel */ }2575};2576 2577MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids);2578 2579static int atmel_sha_probe(struct platform_device *pdev)2580{2581 struct atmel_sha_dev *sha_dd;2582 struct device *dev = &pdev->dev;2583 struct resource *sha_res;2584 int err;2585 2586 sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL);2587 if (!sha_dd)2588 return -ENOMEM;2589 2590 sha_dd->dev = dev;2591 2592 platform_set_drvdata(pdev, sha_dd);2593 2594 INIT_LIST_HEAD(&sha_dd->list);2595 spin_lock_init(&sha_dd->lock);2596 2597 tasklet_init(&sha_dd->done_task, atmel_sha_done_task,2598 (unsigned long)sha_dd);2599 tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task,2600 (unsigned long)sha_dd);2601 2602 crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH);2603 2604 sha_dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &sha_res);2605 if (IS_ERR(sha_dd->io_base)) {2606 err = PTR_ERR(sha_dd->io_base);2607 goto err_tasklet_kill;2608 }2609 sha_dd->phys_base = sha_res->start;2610 2611 /* Get the IRQ */2612 sha_dd->irq = platform_get_irq(pdev, 0);2613 if (sha_dd->irq < 0) {2614 err = sha_dd->irq;2615 goto err_tasklet_kill;2616 }2617 2618 err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq,2619 IRQF_SHARED, "atmel-sha", sha_dd);2620 if (err) {2621 dev_err(dev, "unable to request sha irq.\n");2622 goto err_tasklet_kill;2623 }2624 2625 /* Initializing the clock */2626 sha_dd->iclk = devm_clk_get_prepared(&pdev->dev, "sha_clk");2627 if (IS_ERR(sha_dd->iclk)) {2628 dev_err(dev, "clock initialization failed.\n");2629 err = PTR_ERR(sha_dd->iclk);2630 goto err_tasklet_kill;2631 }2632 2633 err = atmel_sha_hw_version_init(sha_dd);2634 if (err)2635 goto err_tasklet_kill;2636 2637 atmel_sha_get_cap(sha_dd);2638 2639 if (sha_dd->caps.has_dma) {2640 err = atmel_sha_dma_init(sha_dd);2641 if (err)2642 goto err_tasklet_kill;2643 2644 dev_info(dev, "using %s for DMA transfers\n",2645 dma_chan_name(sha_dd->dma_lch_in.chan));2646 }2647 2648 spin_lock(&atmel_sha.lock);2649 list_add_tail(&sha_dd->list, &atmel_sha.dev_list);2650 spin_unlock(&atmel_sha.lock);2651 2652 err = atmel_sha_register_algs(sha_dd);2653 if (err)2654 goto err_algs;2655 2656 dev_info(dev, "Atmel SHA1/SHA256%s%s\n",2657 sha_dd->caps.has_sha224 ? "/SHA224" : "",2658 sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : "");2659 2660 return 0;2661 2662err_algs:2663 spin_lock(&atmel_sha.lock);2664 list_del(&sha_dd->list);2665 spin_unlock(&atmel_sha.lock);2666 if (sha_dd->caps.has_dma)2667 atmel_sha_dma_cleanup(sha_dd);2668err_tasklet_kill:2669 tasklet_kill(&sha_dd->queue_task);2670 tasklet_kill(&sha_dd->done_task);2671 2672 return err;2673}2674 2675static void atmel_sha_remove(struct platform_device *pdev)2676{2677 struct atmel_sha_dev *sha_dd = platform_get_drvdata(pdev);2678 2679 spin_lock(&atmel_sha.lock);2680 list_del(&sha_dd->list);2681 spin_unlock(&atmel_sha.lock);2682 2683 atmel_sha_unregister_algs(sha_dd);2684 2685 tasklet_kill(&sha_dd->queue_task);2686 tasklet_kill(&sha_dd->done_task);2687 2688 if (sha_dd->caps.has_dma)2689 atmel_sha_dma_cleanup(sha_dd);2690}2691 2692static struct platform_driver atmel_sha_driver = {2693 .probe = atmel_sha_probe,2694 .remove_new = atmel_sha_remove,2695 .driver = {2696 .name = "atmel_sha",2697 .of_match_table = atmel_sha_dt_ids,2698 },2699};2700 2701module_platform_driver(atmel_sha_driver);2702 2703MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support.");2704MODULE_LICENSE("GPL v2");2705MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");2706