1100 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2014 Imagination Technologies4 * Authors: Will Thomas, James Hartley5 *6 * Interface structure taken from omap-sham driver7 */8 9#include <linux/clk.h>10#include <linux/dma-mapping.h>11#include <linux/dmaengine.h>12#include <linux/interrupt.h>13#include <linux/io.h>14#include <linux/kernel.h>15#include <linux/module.h>16#include <linux/mod_devicetable.h>17#include <linux/platform_device.h>18#include <linux/scatterlist.h>19 20#include <crypto/internal/hash.h>21#include <crypto/md5.h>22#include <crypto/sha1.h>23#include <crypto/sha2.h>24 25#define CR_RESET 026#define CR_RESET_SET 127#define CR_RESET_UNSET 028 29#define CR_MESSAGE_LENGTH_H 0x430#define CR_MESSAGE_LENGTH_L 0x831 32#define CR_CONTROL 0xc33#define CR_CONTROL_BYTE_ORDER_3210 034#define CR_CONTROL_BYTE_ORDER_0123 135#define CR_CONTROL_BYTE_ORDER_2310 236#define CR_CONTROL_BYTE_ORDER_1032 337#define CR_CONTROL_BYTE_ORDER_SHIFT 838#define CR_CONTROL_ALGO_MD5 039#define CR_CONTROL_ALGO_SHA1 140#define CR_CONTROL_ALGO_SHA224 241#define CR_CONTROL_ALGO_SHA256 342 43#define CR_INTSTAT 0x1044#define CR_INTENAB 0x1445#define CR_INTCLEAR 0x1846#define CR_INT_RESULTS_AVAILABLE BIT(0)47#define CR_INT_NEW_RESULTS_SET BIT(1)48#define CR_INT_RESULT_READ_ERR BIT(2)49#define CR_INT_MESSAGE_WRITE_ERROR BIT(3)50#define CR_INT_STATUS BIT(8)51 52#define CR_RESULT_QUEUE 0x1c53#define CR_RSD0 0x4054#define CR_CORE_REV 0x5055#define CR_CORE_DES1 0x6056#define CR_CORE_DES2 0x7057 58#define DRIVER_FLAGS_BUSY BIT(0)59#define DRIVER_FLAGS_FINAL BIT(1)60#define DRIVER_FLAGS_DMA_ACTIVE BIT(2)61#define DRIVER_FLAGS_OUTPUT_READY BIT(3)62#define DRIVER_FLAGS_INIT BIT(4)63#define DRIVER_FLAGS_CPU BIT(5)64#define DRIVER_FLAGS_DMA_READY BIT(6)65#define DRIVER_FLAGS_ERROR BIT(7)66#define DRIVER_FLAGS_SG BIT(8)67#define DRIVER_FLAGS_SHA1 BIT(18)68#define DRIVER_FLAGS_SHA224 BIT(19)69#define DRIVER_FLAGS_SHA256 BIT(20)70#define DRIVER_FLAGS_MD5 BIT(21)71 72#define IMG_HASH_QUEUE_LENGTH 2073#define IMG_HASH_DMA_BURST 474#define IMG_HASH_DMA_THRESHOLD 6475 76#ifdef __LITTLE_ENDIAN77#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_321078#else79#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_012380#endif81 82struct img_hash_dev;83 84struct img_hash_request_ctx {85 struct img_hash_dev *hdev;86 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));87 unsigned long flags;88 size_t digsize;89 90 dma_addr_t dma_addr;91 size_t dma_ct;92 93 /* sg root */94 struct scatterlist *sgfirst;95 /* walk state */96 struct scatterlist *sg;97 size_t nents;98 size_t offset;99 unsigned int total;100 size_t sent;101 102 unsigned long op;103 104 size_t bufcnt;105 struct ahash_request fallback_req;106 107 /* Zero length buffer must remain last member of struct */108 u8 buffer[] __aligned(sizeof(u32));109};110 111struct img_hash_ctx {112 struct img_hash_dev *hdev;113 unsigned long flags;114 struct crypto_ahash *fallback;115};116 117struct img_hash_dev {118 struct list_head list;119 struct device *dev;120 struct clk *hash_clk;121 struct clk *sys_clk;122 void __iomem *io_base;123 124 phys_addr_t bus_addr;125 void __iomem *cpu_addr;126 127 spinlock_t lock;128 int err;129 struct tasklet_struct done_task;130 struct tasklet_struct dma_task;131 132 unsigned long flags;133 struct crypto_queue queue;134 struct ahash_request *req;135 136 struct dma_chan *dma_lch;137};138 139struct img_hash_drv {140 struct list_head dev_list;141 spinlock_t lock;142};143 144static struct img_hash_drv img_hash = {145 .dev_list = LIST_HEAD_INIT(img_hash.dev_list),146 .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),147};148 149static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)150{151 return readl_relaxed(hdev->io_base + offset);152}153 154static inline void img_hash_write(struct img_hash_dev *hdev,155 u32 offset, u32 value)156{157 writel_relaxed(value, hdev->io_base + offset);158}159 160static inline __be32 img_hash_read_result_queue(struct img_hash_dev *hdev)161{162 return cpu_to_be32(img_hash_read(hdev, CR_RESULT_QUEUE));163}164 165static void img_hash_start(struct img_hash_dev *hdev, bool dma)166{167 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);168 u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;169 170 if (ctx->flags & DRIVER_FLAGS_MD5)171 cr |= CR_CONTROL_ALGO_MD5;172 else if (ctx->flags & DRIVER_FLAGS_SHA1)173 cr |= CR_CONTROL_ALGO_SHA1;174 else if (ctx->flags & DRIVER_FLAGS_SHA224)175 cr |= CR_CONTROL_ALGO_SHA224;176 else if (ctx->flags & DRIVER_FLAGS_SHA256)177 cr |= CR_CONTROL_ALGO_SHA256;178 dev_dbg(hdev->dev, "Starting hash process\n");179 img_hash_write(hdev, CR_CONTROL, cr);180 181 /*182 * The hardware block requires two cycles between writing the control183 * register and writing the first word of data in non DMA mode, to184 * ensure the first data write is not grouped in burst with the control185 * register write a read is issued to 'flush' the bus.186 */187 if (!dma)188 img_hash_read(hdev, CR_CONTROL);189}190 191static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,192 size_t length, int final)193{194 u32 count, len32;195 const u32 *buffer = (const u32 *)buf;196 197 dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length);198 199 if (final)200 hdev->flags |= DRIVER_FLAGS_FINAL;201 202 len32 = DIV_ROUND_UP(length, sizeof(u32));203 204 for (count = 0; count < len32; count++)205 writel_relaxed(buffer[count], hdev->cpu_addr);206 207 return -EINPROGRESS;208}209 210static void img_hash_dma_callback(void *data)211{212 struct img_hash_dev *hdev = data;213 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);214 215 if (ctx->bufcnt) {216 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);217 ctx->bufcnt = 0;218 }219 if (ctx->sg)220 tasklet_schedule(&hdev->dma_task);221}222 223static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)224{225 struct dma_async_tx_descriptor *desc;226 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);227 228 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);229 if (ctx->dma_ct == 0) {230 dev_err(hdev->dev, "Invalid DMA sg\n");231 hdev->err = -EINVAL;232 return -EINVAL;233 }234 235 desc = dmaengine_prep_slave_sg(hdev->dma_lch,236 sg,237 ctx->dma_ct,238 DMA_MEM_TO_DEV,239 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);240 if (!desc) {241 dev_err(hdev->dev, "Null DMA descriptor\n");242 hdev->err = -EINVAL;243 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);244 return -EINVAL;245 }246 desc->callback = img_hash_dma_callback;247 desc->callback_param = hdev;248 dmaengine_submit(desc);249 dma_async_issue_pending(hdev->dma_lch);250 251 return 0;252}253 254static int img_hash_write_via_cpu(struct img_hash_dev *hdev)255{256 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);257 258 ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),259 ctx->buffer, hdev->req->nbytes);260 261 ctx->total = hdev->req->nbytes;262 ctx->bufcnt = 0;263 264 hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);265 266 img_hash_start(hdev, false);267 268 return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);269}270 271static int img_hash_finish(struct ahash_request *req)272{273 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);274 275 if (!req->result)276 return -EINVAL;277 278 memcpy(req->result, ctx->digest, ctx->digsize);279 280 return 0;281}282 283static void img_hash_copy_hash(struct ahash_request *req)284{285 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);286 __be32 *hash = (__be32 *)ctx->digest;287 int i;288 289 for (i = (ctx->digsize / sizeof(*hash)) - 1; i >= 0; i--)290 hash[i] = img_hash_read_result_queue(ctx->hdev);291}292 293static void img_hash_finish_req(struct ahash_request *req, int err)294{295 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);296 struct img_hash_dev *hdev = ctx->hdev;297 298 if (!err) {299 img_hash_copy_hash(req);300 if (DRIVER_FLAGS_FINAL & hdev->flags)301 err = img_hash_finish(req);302 } else {303 dev_warn(hdev->dev, "Hash failed with error %d\n", err);304 ctx->flags |= DRIVER_FLAGS_ERROR;305 }306 307 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |308 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);309 310 if (req->base.complete)311 ahash_request_complete(req, err);312}313 314static int img_hash_write_via_dma(struct img_hash_dev *hdev)315{316 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);317 318 img_hash_start(hdev, true);319 320 dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);321 322 if (!ctx->total)323 hdev->flags |= DRIVER_FLAGS_FINAL;324 325 hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;326 327 tasklet_schedule(&hdev->dma_task);328 329 return -EINPROGRESS;330}331 332static int img_hash_dma_init(struct img_hash_dev *hdev)333{334 struct dma_slave_config dma_conf;335 int err;336 337 hdev->dma_lch = dma_request_chan(hdev->dev, "tx");338 if (IS_ERR(hdev->dma_lch)) {339 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");340 return PTR_ERR(hdev->dma_lch);341 }342 dma_conf.direction = DMA_MEM_TO_DEV;343 dma_conf.dst_addr = hdev->bus_addr;344 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;345 dma_conf.dst_maxburst = IMG_HASH_DMA_BURST;346 dma_conf.device_fc = false;347 348 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);349 if (err) {350 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");351 dma_release_channel(hdev->dma_lch);352 return err;353 }354 355 return 0;356}357 358static void img_hash_dma_task(unsigned long d)359{360 struct img_hash_dev *hdev = (struct img_hash_dev *)d;361 struct img_hash_request_ctx *ctx;362 u8 *addr;363 size_t nbytes, bleft, wsend, len, tbc;364 struct scatterlist tsg;365 366 if (!hdev->req)367 return;368 369 ctx = ahash_request_ctx(hdev->req);370 if (!ctx->sg)371 return;372 373 addr = sg_virt(ctx->sg);374 nbytes = ctx->sg->length - ctx->offset;375 376 /*377 * The hash accelerator does not support a data valid mask. This means378 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the379 * padding bytes in the last word written by that dma would erroneously380 * be included in the hash. To avoid this we round down the transfer,381 * and add the excess to the start of the next dma. It does not matter382 * that the final dma may not be a multiple of 4 bytes as the hashing383 * block is programmed to accept the correct number of bytes.384 */385 386 bleft = nbytes % 4;387 wsend = (nbytes / 4);388 389 if (wsend) {390 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);391 if (img_hash_xmit_dma(hdev, &tsg)) {392 dev_err(hdev->dev, "DMA failed, falling back to CPU");393 ctx->flags |= DRIVER_FLAGS_CPU;394 hdev->err = 0;395 img_hash_xmit_cpu(hdev, addr + ctx->offset,396 wsend * 4, 0);397 ctx->sent += wsend * 4;398 wsend = 0;399 } else {400 ctx->sent += wsend * 4;401 }402 }403 404 if (bleft) {405 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,406 ctx->buffer, bleft, ctx->sent);407 tbc = 0;408 ctx->sg = sg_next(ctx->sg);409 while (ctx->sg && (ctx->bufcnt < 4)) {410 len = ctx->sg->length;411 if (likely(len > (4 - ctx->bufcnt)))412 len = 4 - ctx->bufcnt;413 tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,414 ctx->buffer + ctx->bufcnt, len,415 ctx->sent + ctx->bufcnt);416 ctx->bufcnt += tbc;417 if (tbc >= ctx->sg->length) {418 ctx->sg = sg_next(ctx->sg);419 tbc = 0;420 }421 }422 423 ctx->sent += ctx->bufcnt;424 ctx->offset = tbc;425 426 if (!wsend)427 img_hash_dma_callback(hdev);428 } else {429 ctx->offset = 0;430 ctx->sg = sg_next(ctx->sg);431 }432}433 434static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)435{436 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);437 438 if (ctx->flags & DRIVER_FLAGS_SG)439 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);440 441 return 0;442}443 444static int img_hash_process_data(struct img_hash_dev *hdev)445{446 struct ahash_request *req = hdev->req;447 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);448 int err = 0;449 450 ctx->bufcnt = 0;451 452 if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {453 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",454 req->nbytes);455 err = img_hash_write_via_dma(hdev);456 } else {457 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",458 req->nbytes);459 err = img_hash_write_via_cpu(hdev);460 }461 return err;462}463 464static int img_hash_hw_init(struct img_hash_dev *hdev)465{466 unsigned long long nbits;467 u32 u, l;468 469 img_hash_write(hdev, CR_RESET, CR_RESET_SET);470 img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);471 img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);472 473 nbits = (u64)hdev->req->nbytes << 3;474 u = nbits >> 32;475 l = nbits;476 img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);477 img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);478 479 if (!(DRIVER_FLAGS_INIT & hdev->flags)) {480 hdev->flags |= DRIVER_FLAGS_INIT;481 hdev->err = 0;482 }483 dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);484 return 0;485}486 487static int img_hash_init(struct ahash_request *req)488{489 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);490 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);491 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);492 493 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);494 rctx->fallback_req.base.flags = req->base.flags495 & CRYPTO_TFM_REQ_MAY_SLEEP;496 497 return crypto_ahash_init(&rctx->fallback_req);498}499 500static int img_hash_handle_queue(struct img_hash_dev *hdev,501 struct ahash_request *req)502{503 struct crypto_async_request *async_req, *backlog;504 struct img_hash_request_ctx *ctx;505 unsigned long flags;506 int err = 0, res = 0;507 508 spin_lock_irqsave(&hdev->lock, flags);509 510 if (req)511 res = ahash_enqueue_request(&hdev->queue, req);512 513 if (DRIVER_FLAGS_BUSY & hdev->flags) {514 spin_unlock_irqrestore(&hdev->lock, flags);515 return res;516 }517 518 backlog = crypto_get_backlog(&hdev->queue);519 async_req = crypto_dequeue_request(&hdev->queue);520 if (async_req)521 hdev->flags |= DRIVER_FLAGS_BUSY;522 523 spin_unlock_irqrestore(&hdev->lock, flags);524 525 if (!async_req)526 return res;527 528 if (backlog)529 crypto_request_complete(backlog, -EINPROGRESS);530 531 req = ahash_request_cast(async_req);532 hdev->req = req;533 534 ctx = ahash_request_ctx(req);535 536 dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",537 ctx->op, req->nbytes);538 539 err = img_hash_hw_init(hdev);540 541 if (!err)542 err = img_hash_process_data(hdev);543 544 if (err != -EINPROGRESS) {545 /* done_task will not finish so do it here */546 img_hash_finish_req(req, err);547 }548 return res;549}550 551static int img_hash_update(struct ahash_request *req)552{553 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);554 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);555 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);556 557 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);558 rctx->fallback_req.base.flags = req->base.flags559 & CRYPTO_TFM_REQ_MAY_SLEEP;560 rctx->fallback_req.nbytes = req->nbytes;561 rctx->fallback_req.src = req->src;562 563 return crypto_ahash_update(&rctx->fallback_req);564}565 566static int img_hash_final(struct ahash_request *req)567{568 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);569 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);570 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);571 572 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);573 rctx->fallback_req.base.flags = req->base.flags574 & CRYPTO_TFM_REQ_MAY_SLEEP;575 rctx->fallback_req.result = req->result;576 577 return crypto_ahash_final(&rctx->fallback_req);578}579 580static int img_hash_finup(struct ahash_request *req)581{582 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);583 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);584 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);585 586 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);587 rctx->fallback_req.base.flags = req->base.flags588 & CRYPTO_TFM_REQ_MAY_SLEEP;589 rctx->fallback_req.nbytes = req->nbytes;590 rctx->fallback_req.src = req->src;591 rctx->fallback_req.result = req->result;592 593 return crypto_ahash_finup(&rctx->fallback_req);594}595 596static int img_hash_import(struct ahash_request *req, const void *in)597{598 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);599 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);600 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);601 602 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);603 rctx->fallback_req.base.flags = req->base.flags604 & CRYPTO_TFM_REQ_MAY_SLEEP;605 606 return crypto_ahash_import(&rctx->fallback_req, in);607}608 609static int img_hash_export(struct ahash_request *req, void *out)610{611 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);612 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);613 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);614 615 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);616 rctx->fallback_req.base.flags = req->base.flags617 & CRYPTO_TFM_REQ_MAY_SLEEP;618 619 return crypto_ahash_export(&rctx->fallback_req, out);620}621 622static int img_hash_digest(struct ahash_request *req)623{624 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);625 struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);626 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);627 struct img_hash_dev *hdev = NULL;628 struct img_hash_dev *tmp;629 int err;630 631 spin_lock(&img_hash.lock);632 if (!tctx->hdev) {633 list_for_each_entry(tmp, &img_hash.dev_list, list) {634 hdev = tmp;635 break;636 }637 tctx->hdev = hdev;638 639 } else {640 hdev = tctx->hdev;641 }642 643 spin_unlock(&img_hash.lock);644 ctx->hdev = hdev;645 ctx->flags = 0;646 ctx->digsize = crypto_ahash_digestsize(tfm);647 648 switch (ctx->digsize) {649 case SHA1_DIGEST_SIZE:650 ctx->flags |= DRIVER_FLAGS_SHA1;651 break;652 case SHA256_DIGEST_SIZE:653 ctx->flags |= DRIVER_FLAGS_SHA256;654 break;655 case SHA224_DIGEST_SIZE:656 ctx->flags |= DRIVER_FLAGS_SHA224;657 break;658 case MD5_DIGEST_SIZE:659 ctx->flags |= DRIVER_FLAGS_MD5;660 break;661 default:662 return -EINVAL;663 }664 665 ctx->bufcnt = 0;666 ctx->offset = 0;667 ctx->sent = 0;668 ctx->total = req->nbytes;669 ctx->sg = req->src;670 ctx->sgfirst = req->src;671 ctx->nents = sg_nents(ctx->sg);672 673 err = img_hash_handle_queue(tctx->hdev, req);674 675 return err;676}677 678static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name)679{680 struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);681 682 ctx->fallback = crypto_alloc_ahash(alg_name, 0,683 CRYPTO_ALG_NEED_FALLBACK);684 if (IS_ERR(ctx->fallback)) {685 pr_err("img_hash: Could not load fallback driver.\n");686 return PTR_ERR(ctx->fallback);687 }688 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),689 sizeof(struct img_hash_request_ctx) +690 crypto_ahash_reqsize(ctx->fallback) +691 IMG_HASH_DMA_THRESHOLD);692 693 return 0;694}695 696static int img_hash_cra_md5_init(struct crypto_tfm *tfm)697{698 return img_hash_cra_init(tfm, "md5-generic");699}700 701static int img_hash_cra_sha1_init(struct crypto_tfm *tfm)702{703 return img_hash_cra_init(tfm, "sha1-generic");704}705 706static int img_hash_cra_sha224_init(struct crypto_tfm *tfm)707{708 return img_hash_cra_init(tfm, "sha224-generic");709}710 711static int img_hash_cra_sha256_init(struct crypto_tfm *tfm)712{713 return img_hash_cra_init(tfm, "sha256-generic");714}715 716static void img_hash_cra_exit(struct crypto_tfm *tfm)717{718 struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);719 720 crypto_free_ahash(tctx->fallback);721}722 723static irqreturn_t img_irq_handler(int irq, void *dev_id)724{725 struct img_hash_dev *hdev = dev_id;726 u32 reg;727 728 reg = img_hash_read(hdev, CR_INTSTAT);729 img_hash_write(hdev, CR_INTCLEAR, reg);730 731 if (reg & CR_INT_NEW_RESULTS_SET) {732 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");733 if (DRIVER_FLAGS_BUSY & hdev->flags) {734 hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;735 if (!(DRIVER_FLAGS_CPU & hdev->flags))736 hdev->flags |= DRIVER_FLAGS_DMA_READY;737 tasklet_schedule(&hdev->done_task);738 } else {739 dev_warn(hdev->dev,740 "HASH interrupt when no active requests.\n");741 }742 } else if (reg & CR_INT_RESULTS_AVAILABLE) {743 dev_warn(hdev->dev,744 "IRQ triggered before the hash had completed\n");745 } else if (reg & CR_INT_RESULT_READ_ERR) {746 dev_warn(hdev->dev,747 "Attempt to read from an empty result queue\n");748 } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {749 dev_warn(hdev->dev,750 "Data written before the hardware was configured\n");751 }752 return IRQ_HANDLED;753}754 755static struct ahash_alg img_algs[] = {756 {757 .init = img_hash_init,758 .update = img_hash_update,759 .final = img_hash_final,760 .finup = img_hash_finup,761 .export = img_hash_export,762 .import = img_hash_import,763 .digest = img_hash_digest,764 .halg = {765 .digestsize = MD5_DIGEST_SIZE,766 .statesize = sizeof(struct md5_state),767 .base = {768 .cra_name = "md5",769 .cra_driver_name = "img-md5",770 .cra_priority = 300,771 .cra_flags =772 CRYPTO_ALG_ASYNC |773 CRYPTO_ALG_NEED_FALLBACK,774 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,775 .cra_ctxsize = sizeof(struct img_hash_ctx),776 .cra_init = img_hash_cra_md5_init,777 .cra_exit = img_hash_cra_exit,778 .cra_module = THIS_MODULE,779 }780 }781 },782 {783 .init = img_hash_init,784 .update = img_hash_update,785 .final = img_hash_final,786 .finup = img_hash_finup,787 .export = img_hash_export,788 .import = img_hash_import,789 .digest = img_hash_digest,790 .halg = {791 .digestsize = SHA1_DIGEST_SIZE,792 .statesize = sizeof(struct sha1_state),793 .base = {794 .cra_name = "sha1",795 .cra_driver_name = "img-sha1",796 .cra_priority = 300,797 .cra_flags =798 CRYPTO_ALG_ASYNC |799 CRYPTO_ALG_NEED_FALLBACK,800 .cra_blocksize = SHA1_BLOCK_SIZE,801 .cra_ctxsize = sizeof(struct img_hash_ctx),802 .cra_init = img_hash_cra_sha1_init,803 .cra_exit = img_hash_cra_exit,804 .cra_module = THIS_MODULE,805 }806 }807 },808 {809 .init = img_hash_init,810 .update = img_hash_update,811 .final = img_hash_final,812 .finup = img_hash_finup,813 .export = img_hash_export,814 .import = img_hash_import,815 .digest = img_hash_digest,816 .halg = {817 .digestsize = SHA224_DIGEST_SIZE,818 .statesize = sizeof(struct sha256_state),819 .base = {820 .cra_name = "sha224",821 .cra_driver_name = "img-sha224",822 .cra_priority = 300,823 .cra_flags =824 CRYPTO_ALG_ASYNC |825 CRYPTO_ALG_NEED_FALLBACK,826 .cra_blocksize = SHA224_BLOCK_SIZE,827 .cra_ctxsize = sizeof(struct img_hash_ctx),828 .cra_init = img_hash_cra_sha224_init,829 .cra_exit = img_hash_cra_exit,830 .cra_module = THIS_MODULE,831 }832 }833 },834 {835 .init = img_hash_init,836 .update = img_hash_update,837 .final = img_hash_final,838 .finup = img_hash_finup,839 .export = img_hash_export,840 .import = img_hash_import,841 .digest = img_hash_digest,842 .halg = {843 .digestsize = SHA256_DIGEST_SIZE,844 .statesize = sizeof(struct sha256_state),845 .base = {846 .cra_name = "sha256",847 .cra_driver_name = "img-sha256",848 .cra_priority = 300,849 .cra_flags =850 CRYPTO_ALG_ASYNC |851 CRYPTO_ALG_NEED_FALLBACK,852 .cra_blocksize = SHA256_BLOCK_SIZE,853 .cra_ctxsize = sizeof(struct img_hash_ctx),854 .cra_init = img_hash_cra_sha256_init,855 .cra_exit = img_hash_cra_exit,856 .cra_module = THIS_MODULE,857 }858 }859 }860};861 862static int img_register_algs(struct img_hash_dev *hdev)863{864 int i, err;865 866 for (i = 0; i < ARRAY_SIZE(img_algs); i++) {867 err = crypto_register_ahash(&img_algs[i]);868 if (err)869 goto err_reg;870 }871 return 0;872 873err_reg:874 for (; i--; )875 crypto_unregister_ahash(&img_algs[i]);876 877 return err;878}879 880static int img_unregister_algs(struct img_hash_dev *hdev)881{882 int i;883 884 for (i = 0; i < ARRAY_SIZE(img_algs); i++)885 crypto_unregister_ahash(&img_algs[i]);886 return 0;887}888 889static void img_hash_done_task(unsigned long data)890{891 struct img_hash_dev *hdev = (struct img_hash_dev *)data;892 int err = 0;893 894 if (hdev->err == -EINVAL) {895 err = hdev->err;896 goto finish;897 }898 899 if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {900 img_hash_handle_queue(hdev, NULL);901 return;902 }903 904 if (DRIVER_FLAGS_CPU & hdev->flags) {905 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {906 hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;907 goto finish;908 }909 } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {910 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {911 hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;912 img_hash_write_via_dma_stop(hdev);913 if (hdev->err) {914 err = hdev->err;915 goto finish;916 }917 }918 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {919 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |920 DRIVER_FLAGS_OUTPUT_READY);921 goto finish;922 }923 }924 return;925 926finish:927 img_hash_finish_req(hdev->req, err);928}929 930static const struct of_device_id img_hash_match[] __maybe_unused = {931 { .compatible = "img,hash-accelerator" },932 {}933};934MODULE_DEVICE_TABLE(of, img_hash_match);935 936static int img_hash_probe(struct platform_device *pdev)937{938 struct img_hash_dev *hdev;939 struct device *dev = &pdev->dev;940 struct resource *hash_res;941 int irq;942 int err;943 944 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);945 if (hdev == NULL)946 return -ENOMEM;947 948 spin_lock_init(&hdev->lock);949 950 hdev->dev = dev;951 952 platform_set_drvdata(pdev, hdev);953 954 INIT_LIST_HEAD(&hdev->list);955 956 tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);957 tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);958 959 crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);960 961 /* Register bank */962 hdev->io_base = devm_platform_ioremap_resource(pdev, 0);963 if (IS_ERR(hdev->io_base)) {964 err = PTR_ERR(hdev->io_base);965 goto res_err;966 }967 968 /* Write port (DMA or CPU) */969 hdev->cpu_addr = devm_platform_get_and_ioremap_resource(pdev, 1, &hash_res);970 if (IS_ERR(hdev->cpu_addr)) {971 err = PTR_ERR(hdev->cpu_addr);972 goto res_err;973 }974 hdev->bus_addr = hash_res->start;975 976 irq = platform_get_irq(pdev, 0);977 if (irq < 0) {978 err = irq;979 goto res_err;980 }981 982 err = devm_request_irq(dev, irq, img_irq_handler, 0,983 dev_name(dev), hdev);984 if (err) {985 dev_err(dev, "unable to request irq\n");986 goto res_err;987 }988 dev_dbg(dev, "using IRQ channel %d\n", irq);989 990 hdev->hash_clk = devm_clk_get_enabled(&pdev->dev, "hash");991 if (IS_ERR(hdev->hash_clk)) {992 dev_err(dev, "clock initialization failed.\n");993 err = PTR_ERR(hdev->hash_clk);994 goto res_err;995 }996 997 hdev->sys_clk = devm_clk_get_enabled(&pdev->dev, "sys");998 if (IS_ERR(hdev->sys_clk)) {999 dev_err(dev, "clock initialization failed.\n");1000 err = PTR_ERR(hdev->sys_clk);1001 goto res_err;1002 }1003 1004 err = img_hash_dma_init(hdev);1005 if (err)1006 goto res_err;1007 1008 dev_dbg(dev, "using %s for DMA transfers\n",1009 dma_chan_name(hdev->dma_lch));1010 1011 spin_lock(&img_hash.lock);1012 list_add_tail(&hdev->list, &img_hash.dev_list);1013 spin_unlock(&img_hash.lock);1014 1015 err = img_register_algs(hdev);1016 if (err)1017 goto err_algs;1018 dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");1019 1020 return 0;1021 1022err_algs:1023 spin_lock(&img_hash.lock);1024 list_del(&hdev->list);1025 spin_unlock(&img_hash.lock);1026 dma_release_channel(hdev->dma_lch);1027res_err:1028 tasklet_kill(&hdev->done_task);1029 tasklet_kill(&hdev->dma_task);1030 1031 return err;1032}1033 1034static void img_hash_remove(struct platform_device *pdev)1035{1036 struct img_hash_dev *hdev;1037 1038 hdev = platform_get_drvdata(pdev);1039 spin_lock(&img_hash.lock);1040 list_del(&hdev->list);1041 spin_unlock(&img_hash.lock);1042 1043 img_unregister_algs(hdev);1044 1045 tasklet_kill(&hdev->done_task);1046 tasklet_kill(&hdev->dma_task);1047 1048 dma_release_channel(hdev->dma_lch);1049}1050 1051#ifdef CONFIG_PM_SLEEP1052static int img_hash_suspend(struct device *dev)1053{1054 struct img_hash_dev *hdev = dev_get_drvdata(dev);1055 1056 clk_disable_unprepare(hdev->hash_clk);1057 clk_disable_unprepare(hdev->sys_clk);1058 1059 return 0;1060}1061 1062static int img_hash_resume(struct device *dev)1063{1064 struct img_hash_dev *hdev = dev_get_drvdata(dev);1065 int ret;1066 1067 ret = clk_prepare_enable(hdev->hash_clk);1068 if (ret)1069 return ret;1070 1071 ret = clk_prepare_enable(hdev->sys_clk);1072 if (ret) {1073 clk_disable_unprepare(hdev->hash_clk);1074 return ret;1075 }1076 1077 return 0;1078}1079#endif /* CONFIG_PM_SLEEP */1080 1081static const struct dev_pm_ops img_hash_pm_ops = {1082 SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume)1083};1084 1085static struct platform_driver img_hash_driver = {1086 .probe = img_hash_probe,1087 .remove_new = img_hash_remove,1088 .driver = {1089 .name = "img-hash-accelerator",1090 .pm = &img_hash_pm_ops,1091 .of_match_table = img_hash_match,1092 }1093};1094module_platform_driver(img_hash_driver);1095 1096MODULE_LICENSE("GPL v2");1097MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");1098MODULE_AUTHOR("Will Thomas.");1099MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");1100