2351 lines · c
1// SPDX-License-Identifier: GPL-2.02//3// Cryptographic API.4//5// Support for Samsung S5PV210 and Exynos HW acceleration.6//7// Copyright (C) 2011 NetUP Inc. All rights reserved.8// Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved.9//10// Hash part based on omap-sham.c driver.11 12#include <linux/clk.h>13#include <linux/crypto.h>14#include <linux/dma-mapping.h>15#include <linux/err.h>16#include <linux/errno.h>17#include <linux/init.h>18#include <linux/interrupt.h>19#include <linux/io.h>20#include <linux/kernel.h>21#include <linux/module.h>22#include <linux/of.h>23#include <linux/platform_device.h>24#include <linux/scatterlist.h>25 26#include <crypto/ctr.h>27#include <crypto/aes.h>28#include <crypto/algapi.h>29#include <crypto/scatterwalk.h>30 31#include <crypto/hash.h>32#include <crypto/md5.h>33#include <crypto/sha1.h>34#include <crypto/sha2.h>35#include <crypto/internal/hash.h>36 37#define _SBF(s, v) ((v) << (s))38 39/* Feed control registers */40#define SSS_REG_FCINTSTAT 0x000041#define SSS_FCINTSTAT_HPARTINT BIT(7)42#define SSS_FCINTSTAT_HDONEINT BIT(5)43#define SSS_FCINTSTAT_BRDMAINT BIT(3)44#define SSS_FCINTSTAT_BTDMAINT BIT(2)45#define SSS_FCINTSTAT_HRDMAINT BIT(1)46#define SSS_FCINTSTAT_PKDMAINT BIT(0)47 48#define SSS_REG_FCINTENSET 0x000449#define SSS_FCINTENSET_HPARTINTENSET BIT(7)50#define SSS_FCINTENSET_HDONEINTENSET BIT(5)51#define SSS_FCINTENSET_BRDMAINTENSET BIT(3)52#define SSS_FCINTENSET_BTDMAINTENSET BIT(2)53#define SSS_FCINTENSET_HRDMAINTENSET BIT(1)54#define SSS_FCINTENSET_PKDMAINTENSET BIT(0)55 56#define SSS_REG_FCINTENCLR 0x000857#define SSS_FCINTENCLR_HPARTINTENCLR BIT(7)58#define SSS_FCINTENCLR_HDONEINTENCLR BIT(5)59#define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3)60#define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2)61#define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1)62#define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0)63 64#define SSS_REG_FCINTPEND 0x000C65#define SSS_FCINTPEND_HPARTINTP BIT(7)66#define SSS_FCINTPEND_HDONEINTP BIT(5)67#define SSS_FCINTPEND_BRDMAINTP BIT(3)68#define SSS_FCINTPEND_BTDMAINTP BIT(2)69#define SSS_FCINTPEND_HRDMAINTP BIT(1)70#define SSS_FCINTPEND_PKDMAINTP BIT(0)71 72#define SSS_REG_FCFIFOSTAT 0x001073#define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7)74#define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6)75#define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5)76#define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4)77#define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3)78#define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2)79#define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1)80#define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0)81 82#define SSS_REG_FCFIFOCTRL 0x001483#define SSS_FCFIFOCTRL_DESSEL BIT(2)84#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00)85#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01)86#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02)87#define SSS_HASHIN_MASK _SBF(0, 0x03)88 89#define SSS_REG_FCBRDMAS 0x002090#define SSS_REG_FCBRDMAL 0x002491#define SSS_REG_FCBRDMAC 0x002892#define SSS_FCBRDMAC_BYTESWAP BIT(1)93#define SSS_FCBRDMAC_FLUSH BIT(0)94 95#define SSS_REG_FCBTDMAS 0x003096#define SSS_REG_FCBTDMAL 0x003497#define SSS_REG_FCBTDMAC 0x003898#define SSS_FCBTDMAC_BYTESWAP BIT(1)99#define SSS_FCBTDMAC_FLUSH BIT(0)100 101#define SSS_REG_FCHRDMAS 0x0040102#define SSS_REG_FCHRDMAL 0x0044103#define SSS_REG_FCHRDMAC 0x0048104#define SSS_FCHRDMAC_BYTESWAP BIT(1)105#define SSS_FCHRDMAC_FLUSH BIT(0)106 107#define SSS_REG_FCPKDMAS 0x0050108#define SSS_REG_FCPKDMAL 0x0054109#define SSS_REG_FCPKDMAC 0x0058110#define SSS_FCPKDMAC_BYTESWAP BIT(3)111#define SSS_FCPKDMAC_DESCEND BIT(2)112#define SSS_FCPKDMAC_TRANSMIT BIT(1)113#define SSS_FCPKDMAC_FLUSH BIT(0)114 115#define SSS_REG_FCPKDMAO 0x005C116 117/* AES registers */118#define SSS_REG_AES_CONTROL 0x00119#define SSS_AES_BYTESWAP_DI BIT(11)120#define SSS_AES_BYTESWAP_DO BIT(10)121#define SSS_AES_BYTESWAP_IV BIT(9)122#define SSS_AES_BYTESWAP_CNT BIT(8)123#define SSS_AES_BYTESWAP_KEY BIT(7)124#define SSS_AES_KEY_CHANGE_MODE BIT(6)125#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00)126#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01)127#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02)128#define SSS_AES_FIFO_MODE BIT(3)129#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00)130#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01)131#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02)132#define SSS_AES_MODE_DECRYPT BIT(0)133 134#define SSS_REG_AES_STATUS 0x04135#define SSS_AES_BUSY BIT(2)136#define SSS_AES_INPUT_READY BIT(1)137#define SSS_AES_OUTPUT_READY BIT(0)138 139#define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2))140#define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2))141#define SSS_REG_AES_IV_DATA(s) (0x30 + (s << 2))142#define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2))143#define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2))144 145#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg))146#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg))147#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg))148 149#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg)150#define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \151 SSS_AES_REG(dev, reg))152 153/* HW engine modes */154#define FLAGS_AES_DECRYPT BIT(0)155#define FLAGS_AES_MODE_MASK _SBF(1, 0x03)156#define FLAGS_AES_CBC _SBF(1, 0x01)157#define FLAGS_AES_CTR _SBF(1, 0x02)158 159#define AES_KEY_LEN 16160#define CRYPTO_QUEUE_LEN 1161 162/* HASH registers */163#define SSS_REG_HASH_CTRL 0x00164 165#define SSS_HASH_USER_IV_EN BIT(5)166#define SSS_HASH_INIT_BIT BIT(4)167#define SSS_HASH_ENGINE_SHA1 _SBF(1, 0x00)168#define SSS_HASH_ENGINE_MD5 _SBF(1, 0x01)169#define SSS_HASH_ENGINE_SHA256 _SBF(1, 0x02)170 171#define SSS_HASH_ENGINE_MASK _SBF(1, 0x03)172 173#define SSS_REG_HASH_CTRL_PAUSE 0x04174 175#define SSS_HASH_PAUSE BIT(0)176 177#define SSS_REG_HASH_CTRL_FIFO 0x08178 179#define SSS_HASH_FIFO_MODE_DMA BIT(0)180#define SSS_HASH_FIFO_MODE_CPU 0181 182#define SSS_REG_HASH_CTRL_SWAP 0x0C183 184#define SSS_HASH_BYTESWAP_DI BIT(3)185#define SSS_HASH_BYTESWAP_DO BIT(2)186#define SSS_HASH_BYTESWAP_IV BIT(1)187#define SSS_HASH_BYTESWAP_KEY BIT(0)188 189#define SSS_REG_HASH_STATUS 0x10190 191#define SSS_HASH_STATUS_MSG_DONE BIT(6)192#define SSS_HASH_STATUS_PARTIAL_DONE BIT(4)193#define SSS_HASH_STATUS_BUFFER_READY BIT(0)194 195#define SSS_REG_HASH_MSG_SIZE_LOW 0x20196#define SSS_REG_HASH_MSG_SIZE_HIGH 0x24197 198#define SSS_REG_HASH_PRE_MSG_SIZE_LOW 0x28199#define SSS_REG_HASH_PRE_MSG_SIZE_HIGH 0x2C200 201#define SSS_REG_HASH_IV(s) (0xB0 + ((s) << 2))202#define SSS_REG_HASH_OUT(s) (0x100 + ((s) << 2))203 204#define HASH_BLOCK_SIZE 64205#define HASH_REG_SIZEOF 4206#define HASH_MD5_MAX_REG (MD5_DIGEST_SIZE / HASH_REG_SIZEOF)207#define HASH_SHA1_MAX_REG (SHA1_DIGEST_SIZE / HASH_REG_SIZEOF)208#define HASH_SHA256_MAX_REG (SHA256_DIGEST_SIZE / HASH_REG_SIZEOF)209 210/*211 * HASH bit numbers, used by device, setting in dev->hash_flags with212 * functions set_bit(), clear_bit() or tested with test_bit() or BIT(),213 * to keep HASH state BUSY or FREE, or to signal state from irq_handler214 * to hash_tasklet. SGS keep track of allocated memory for scatterlist215 */216#define HASH_FLAGS_BUSY 0217#define HASH_FLAGS_FINAL 1218#define HASH_FLAGS_DMA_ACTIVE 2219#define HASH_FLAGS_OUTPUT_READY 3220#define HASH_FLAGS_DMA_READY 4221#define HASH_FLAGS_SGS_COPIED 5222#define HASH_FLAGS_SGS_ALLOCED 6223 224/* HASH HW constants */225#define BUFLEN HASH_BLOCK_SIZE226 227#define SSS_HASH_QUEUE_LENGTH 10228 229/**230 * struct samsung_aes_variant - platform specific SSS driver data231 * @aes_offset: AES register offset from SSS module's base.232 * @hash_offset: HASH register offset from SSS module's base.233 * @clk_names: names of clocks needed to run SSS IP234 *235 * Specifies platform specific configuration of SSS module.236 * Note: A structure for driver specific platform data is used for future237 * expansion of its usage.238 */239struct samsung_aes_variant {240 unsigned int aes_offset;241 unsigned int hash_offset;242 const char *clk_names[2];243};244 245struct s5p_aes_reqctx {246 unsigned long mode;247};248 249struct s5p_aes_ctx {250 struct s5p_aes_dev *dev;251 252 u8 aes_key[AES_MAX_KEY_SIZE];253 u8 nonce[CTR_RFC3686_NONCE_SIZE];254 int keylen;255};256 257/**258 * struct s5p_aes_dev - Crypto device state container259 * @dev: Associated device260 * @clk: Clock for accessing hardware261 * @pclk: APB bus clock necessary to access the hardware262 * @ioaddr: Mapped IO memory region263 * @aes_ioaddr: Per-varian offset for AES block IO memory264 * @irq_fc: Feed control interrupt line265 * @req: Crypto request currently handled by the device266 * @ctx: Configuration for currently handled crypto request267 * @sg_src: Scatter list with source data for currently handled block268 * in device. This is DMA-mapped into device.269 * @sg_dst: Scatter list with destination data for currently handled block270 * in device. This is DMA-mapped into device.271 * @sg_src_cpy: In case of unaligned access, copied scatter list272 * with source data.273 * @sg_dst_cpy: In case of unaligned access, copied scatter list274 * with destination data.275 * @tasklet: New request scheduling jib276 * @queue: Crypto queue277 * @busy: Indicates whether the device is currently handling some request278 * thus it uses some of the fields from this state, like:279 * req, ctx, sg_src/dst (and copies). This essentially280 * protects against concurrent access to these fields.281 * @lock: Lock for protecting both access to device hardware registers282 * and fields related to current request (including the busy field).283 * @res: Resources for hash.284 * @io_hash_base: Per-variant offset for HASH block IO memory.285 * @hash_lock: Lock for protecting hash_req, hash_queue and hash_flags286 * variable.287 * @hash_flags: Flags for current HASH op.288 * @hash_queue: Async hash queue.289 * @hash_tasklet: New HASH request scheduling job.290 * @xmit_buf: Buffer for current HASH request transfer into SSS block.291 * @hash_req: Current request sending to SSS HASH block.292 * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block.293 * @hash_sg_cnt: Counter for hash_sg_iter.294 *295 * @use_hash: true if HASH algs enabled296 */297struct s5p_aes_dev {298 struct device *dev;299 struct clk *clk;300 struct clk *pclk;301 void __iomem *ioaddr;302 void __iomem *aes_ioaddr;303 int irq_fc;304 305 struct skcipher_request *req;306 struct s5p_aes_ctx *ctx;307 struct scatterlist *sg_src;308 struct scatterlist *sg_dst;309 310 struct scatterlist *sg_src_cpy;311 struct scatterlist *sg_dst_cpy;312 313 struct tasklet_struct tasklet;314 struct crypto_queue queue;315 bool busy;316 spinlock_t lock;317 318 struct resource *res;319 void __iomem *io_hash_base;320 321 spinlock_t hash_lock; /* protect hash_ vars */322 unsigned long hash_flags;323 struct crypto_queue hash_queue;324 struct tasklet_struct hash_tasklet;325 326 u8 xmit_buf[BUFLEN];327 struct ahash_request *hash_req;328 struct scatterlist *hash_sg_iter;329 unsigned int hash_sg_cnt;330 331 bool use_hash;332};333 334/**335 * struct s5p_hash_reqctx - HASH request context336 * @dd: Associated device337 * @op_update: Current request operation (OP_UPDATE or OP_FINAL)338 * @digcnt: Number of bytes processed by HW (without buffer[] ones)339 * @digest: Digest message or IV for partial result340 * @nregs: Number of HW registers for digest or IV read/write341 * @engine: Bits for selecting type of HASH in SSS block342 * @sg: sg for DMA transfer343 * @sg_len: Length of sg for DMA transfer344 * @sgl: sg for joining buffer and req->src scatterlist345 * @skip: Skip offset in req->src for current op346 * @total: Total number of bytes for current request347 * @finup: Keep state for finup or final.348 * @error: Keep track of error.349 * @bufcnt: Number of bytes holded in buffer[]350 * @buffer: For byte(s) from end of req->src in UPDATE op351 */352struct s5p_hash_reqctx {353 struct s5p_aes_dev *dd;354 bool op_update;355 356 u64 digcnt;357 u8 digest[SHA256_DIGEST_SIZE];358 359 unsigned int nregs; /* digest_size / sizeof(reg) */360 u32 engine;361 362 struct scatterlist *sg;363 unsigned int sg_len;364 struct scatterlist sgl[2];365 unsigned int skip;366 unsigned int total;367 bool finup;368 bool error;369 370 u32 bufcnt;371 u8 buffer[];372};373 374/**375 * struct s5p_hash_ctx - HASH transformation context376 * @dd: Associated device377 * @flags: Bits for algorithm HASH.378 * @fallback: Software transformation for zero message or size < BUFLEN.379 */380struct s5p_hash_ctx {381 struct s5p_aes_dev *dd;382 unsigned long flags;383 struct crypto_shash *fallback;384};385 386static const struct samsung_aes_variant s5p_aes_data = {387 .aes_offset = 0x4000,388 .hash_offset = 0x6000,389 .clk_names = { "secss", },390};391 392static const struct samsung_aes_variant exynos_aes_data = {393 .aes_offset = 0x200,394 .hash_offset = 0x400,395 .clk_names = { "secss", },396};397 398static const struct samsung_aes_variant exynos5433_slim_aes_data = {399 .aes_offset = 0x400,400 .hash_offset = 0x800,401 .clk_names = { "aclk", "pclk", },402};403 404static const struct of_device_id s5p_sss_dt_match[] = {405 {406 .compatible = "samsung,s5pv210-secss",407 .data = &s5p_aes_data,408 },409 {410 .compatible = "samsung,exynos4210-secss",411 .data = &exynos_aes_data,412 },413 {414 .compatible = "samsung,exynos5433-slim-sss",415 .data = &exynos5433_slim_aes_data,416 },417 { },418};419MODULE_DEVICE_TABLE(of, s5p_sss_dt_match);420 421static inline const struct samsung_aes_variant *find_s5p_sss_version422 (const struct platform_device *pdev)423{424 if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node))425 return of_device_get_match_data(&pdev->dev);426 427 return (const struct samsung_aes_variant *)428 platform_get_device_id(pdev)->driver_data;429}430 431static struct s5p_aes_dev *s5p_dev;432 433static void s5p_set_dma_indata(struct s5p_aes_dev *dev,434 const struct scatterlist *sg)435{436 SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg));437 SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg));438}439 440static void s5p_set_dma_outdata(struct s5p_aes_dev *dev,441 const struct scatterlist *sg)442{443 SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg));444 SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg));445}446 447static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg)448{449 int len;450 451 if (!*sg)452 return;453 454 len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE);455 free_pages((unsigned long)sg_virt(*sg), get_order(len));456 457 kfree(*sg);458 *sg = NULL;459}460 461static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg,462 unsigned int nbytes, int out)463{464 struct scatter_walk walk;465 466 if (!nbytes)467 return;468 469 scatterwalk_start(&walk, sg);470 scatterwalk_copychunks(buf, &walk, nbytes, out);471 scatterwalk_done(&walk, out, 0);472}473 474static void s5p_sg_done(struct s5p_aes_dev *dev)475{476 struct skcipher_request *req = dev->req;477 struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req);478 479 if (dev->sg_dst_cpy) {480 dev_dbg(dev->dev,481 "Copying %d bytes of output data back to original place\n",482 dev->req->cryptlen);483 s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst,484 dev->req->cryptlen, 1);485 }486 s5p_free_sg_cpy(dev, &dev->sg_src_cpy);487 s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);488 if (reqctx->mode & FLAGS_AES_CBC)489 memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), AES_BLOCK_SIZE);490 491 else if (reqctx->mode & FLAGS_AES_CTR)492 memcpy_fromio(req->iv, dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), AES_BLOCK_SIZE);493}494 495/* Calls the completion. Cannot be called with dev->lock hold. */496static void s5p_aes_complete(struct skcipher_request *req, int err)497{498 skcipher_request_complete(req, err);499}500 501static void s5p_unset_outdata(struct s5p_aes_dev *dev)502{503 dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE);504}505 506static void s5p_unset_indata(struct s5p_aes_dev *dev)507{508 dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE);509}510 511static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,512 struct scatterlist **dst)513{514 void *pages;515 int len;516 517 *dst = kmalloc(sizeof(**dst), GFP_ATOMIC);518 if (!*dst)519 return -ENOMEM;520 521 len = ALIGN(dev->req->cryptlen, AES_BLOCK_SIZE);522 pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len));523 if (!pages) {524 kfree(*dst);525 *dst = NULL;526 return -ENOMEM;527 }528 529 s5p_sg_copy_buf(pages, src, dev->req->cryptlen, 0);530 531 sg_init_table(*dst, 1);532 sg_set_buf(*dst, pages, len);533 534 return 0;535}536 537static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)538{539 if (!sg->length)540 return -EINVAL;541 542 if (!dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE))543 return -ENOMEM;544 545 dev->sg_dst = sg;546 547 return 0;548}549 550static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)551{552 if (!sg->length)553 return -EINVAL;554 555 if (!dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE))556 return -ENOMEM;557 558 dev->sg_src = sg;559 560 return 0;561}562 563/*564 * Returns -ERRNO on error (mapping of new data failed).565 * On success returns:566 * - 0 if there is no more data,567 * - 1 if new transmitting (output) data is ready and its address+length568 * have to be written to device (by calling s5p_set_dma_outdata()).569 */570static int s5p_aes_tx(struct s5p_aes_dev *dev)571{572 int ret = 0;573 574 s5p_unset_outdata(dev);575 576 if (!sg_is_last(dev->sg_dst)) {577 ret = s5p_set_outdata(dev, sg_next(dev->sg_dst));578 if (!ret)579 ret = 1;580 }581 582 return ret;583}584 585/*586 * Returns -ERRNO on error (mapping of new data failed).587 * On success returns:588 * - 0 if there is no more data,589 * - 1 if new receiving (input) data is ready and its address+length590 * have to be written to device (by calling s5p_set_dma_indata()).591 */592static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/)593{594 int ret = 0;595 596 s5p_unset_indata(dev);597 598 if (!sg_is_last(dev->sg_src)) {599 ret = s5p_set_indata(dev, sg_next(dev->sg_src));600 if (!ret)601 ret = 1;602 }603 604 return ret;605}606 607static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset)608{609 return __raw_readl(dd->io_hash_base + offset);610}611 612static inline void s5p_hash_write(struct s5p_aes_dev *dd,613 u32 offset, u32 value)614{615 __raw_writel(value, dd->io_hash_base + offset);616}617 618/**619 * s5p_set_dma_hashdata() - start DMA with sg620 * @dev: device621 * @sg: scatterlist ready to DMA transmit622 */623static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev,624 const struct scatterlist *sg)625{626 dev->hash_sg_cnt--;627 SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg));628 SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */629}630 631/**632 * s5p_hash_rx() - get next hash_sg_iter633 * @dev: device634 *635 * Return:636 * 2 if there is no more data and it is UPDATE op637 * 1 if new receiving (input) data is ready and can be written to device638 * 0 if there is no more data and it is FINAL op639 */640static int s5p_hash_rx(struct s5p_aes_dev *dev)641{642 if (dev->hash_sg_cnt > 0) {643 dev->hash_sg_iter = sg_next(dev->hash_sg_iter);644 return 1;645 }646 647 set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags);648 if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags))649 return 0;650 651 return 2;652}653 654static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)655{656 struct platform_device *pdev = dev_id;657 struct s5p_aes_dev *dev = platform_get_drvdata(pdev);658 struct skcipher_request *req;659 int err_dma_tx = 0;660 int err_dma_rx = 0;661 int err_dma_hx = 0;662 bool tx_end = false;663 bool hx_end = false;664 unsigned long flags;665 u32 status, st_bits;666 int err;667 668 spin_lock_irqsave(&dev->lock, flags);669 670 /*671 * Handle rx or tx interrupt. If there is still data (scatterlist did not672 * reach end), then map next scatterlist entry.673 * In case of such mapping error, s5p_aes_complete() should be called.674 *675 * If there is no more data in tx scatter list, call s5p_aes_complete()676 * and schedule new tasklet.677 *678 * Handle hx interrupt. If there is still data map next entry.679 */680 status = SSS_READ(dev, FCINTSTAT);681 if (status & SSS_FCINTSTAT_BRDMAINT)682 err_dma_rx = s5p_aes_rx(dev);683 684 if (status & SSS_FCINTSTAT_BTDMAINT) {685 if (sg_is_last(dev->sg_dst))686 tx_end = true;687 err_dma_tx = s5p_aes_tx(dev);688 }689 690 if (status & SSS_FCINTSTAT_HRDMAINT)691 err_dma_hx = s5p_hash_rx(dev);692 693 st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT |694 SSS_FCINTSTAT_HRDMAINT);695 /* clear DMA bits */696 SSS_WRITE(dev, FCINTPEND, st_bits);697 698 /* clear HASH irq bits */699 if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) {700 /* cannot have both HPART and HDONE */701 if (status & SSS_FCINTSTAT_HPARTINT)702 st_bits = SSS_HASH_STATUS_PARTIAL_DONE;703 704 if (status & SSS_FCINTSTAT_HDONEINT)705 st_bits = SSS_HASH_STATUS_MSG_DONE;706 707 set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags);708 s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits);709 hx_end = true;710 /* when DONE or PART, do not handle HASH DMA */711 err_dma_hx = 0;712 }713 714 if (err_dma_rx < 0) {715 err = err_dma_rx;716 goto error;717 }718 if (err_dma_tx < 0) {719 err = err_dma_tx;720 goto error;721 }722 723 if (tx_end) {724 s5p_sg_done(dev);725 if (err_dma_hx == 1)726 s5p_set_dma_hashdata(dev, dev->hash_sg_iter);727 728 spin_unlock_irqrestore(&dev->lock, flags);729 730 s5p_aes_complete(dev->req, 0);731 /* Device is still busy */732 tasklet_schedule(&dev->tasklet);733 } else {734 /*735 * Writing length of DMA block (either receiving or736 * transmitting) will start the operation immediately, so this737 * should be done at the end (even after clearing pending738 * interrupts to not miss the interrupt).739 */740 if (err_dma_tx == 1)741 s5p_set_dma_outdata(dev, dev->sg_dst);742 if (err_dma_rx == 1)743 s5p_set_dma_indata(dev, dev->sg_src);744 if (err_dma_hx == 1)745 s5p_set_dma_hashdata(dev, dev->hash_sg_iter);746 747 spin_unlock_irqrestore(&dev->lock, flags);748 }749 750 goto hash_irq_end;751 752error:753 s5p_sg_done(dev);754 dev->busy = false;755 req = dev->req;756 if (err_dma_hx == 1)757 s5p_set_dma_hashdata(dev, dev->hash_sg_iter);758 759 spin_unlock_irqrestore(&dev->lock, flags);760 s5p_aes_complete(req, err);761 762hash_irq_end:763 /*764 * Note about else if:765 * when hash_sg_iter reaches end and its UPDATE op,766 * issue SSS_HASH_PAUSE and wait for HPART irq767 */768 if (hx_end)769 tasklet_schedule(&dev->hash_tasklet);770 else if (err_dma_hx == 2)771 s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE,772 SSS_HASH_PAUSE);773 774 return IRQ_HANDLED;775}776 777/**778 * s5p_hash_read_msg() - read message or IV from HW779 * @req: AHASH request780 */781static void s5p_hash_read_msg(struct ahash_request *req)782{783 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);784 struct s5p_aes_dev *dd = ctx->dd;785 u32 *hash = (u32 *)ctx->digest;786 unsigned int i;787 788 for (i = 0; i < ctx->nregs; i++)789 hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i));790}791 792/**793 * s5p_hash_write_ctx_iv() - write IV for next partial/finup op.794 * @dd: device795 * @ctx: request context796 */797static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd,798 const struct s5p_hash_reqctx *ctx)799{800 const u32 *hash = (const u32 *)ctx->digest;801 unsigned int i;802 803 for (i = 0; i < ctx->nregs; i++)804 s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]);805}806 807/**808 * s5p_hash_write_iv() - write IV for next partial/finup op.809 * @req: AHASH request810 */811static void s5p_hash_write_iv(struct ahash_request *req)812{813 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);814 815 s5p_hash_write_ctx_iv(ctx->dd, ctx);816}817 818/**819 * s5p_hash_copy_result() - copy digest into req->result820 * @req: AHASH request821 */822static void s5p_hash_copy_result(struct ahash_request *req)823{824 const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);825 826 if (!req->result)827 return;828 829 memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF);830}831 832/**833 * s5p_hash_dma_flush() - flush HASH DMA834 * @dev: secss device835 */836static void s5p_hash_dma_flush(struct s5p_aes_dev *dev)837{838 SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH);839}840 841/**842 * s5p_hash_dma_enable() - enable DMA mode for HASH843 * @dev: secss device844 *845 * enable DMA mode for HASH846 */847static void s5p_hash_dma_enable(struct s5p_aes_dev *dev)848{849 s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA);850}851 852/**853 * s5p_hash_irq_disable() - disable irq HASH signals854 * @dev: secss device855 * @flags: bitfield with irq's to be disabled856 */857static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags)858{859 SSS_WRITE(dev, FCINTENCLR, flags);860}861 862/**863 * s5p_hash_irq_enable() - enable irq signals864 * @dev: secss device865 * @flags: bitfield with irq's to be enabled866 */867static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags)868{869 SSS_WRITE(dev, FCINTENSET, flags);870}871 872/**873 * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH874 * @dev: secss device875 * @hashflow: HASH stream flow with/without crypto AES/DES876 */877static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow)878{879 unsigned long flags;880 u32 flow;881 882 spin_lock_irqsave(&dev->lock, flags);883 884 flow = SSS_READ(dev, FCFIFOCTRL);885 flow &= ~SSS_HASHIN_MASK;886 flow |= hashflow;887 SSS_WRITE(dev, FCFIFOCTRL, flow);888 889 spin_unlock_irqrestore(&dev->lock, flags);890}891 892/**893 * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS894 * @dev: secss device895 * @hashflow: HASH stream flow with/without AES/DES896 *897 * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW,898 * enable HASH irq's HRDMA, HDONE, HPART899 */900static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow)901{902 s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR |903 SSS_FCINTENCLR_HDONEINTENCLR |904 SSS_FCINTENCLR_HPARTINTENCLR);905 s5p_hash_dma_flush(dev);906 907 s5p_hash_dma_enable(dev);908 s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK);909 s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET |910 SSS_FCINTENSET_HDONEINTENSET |911 SSS_FCINTENSET_HPARTINTENSET);912}913 914/**915 * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing916 * @dd: secss device917 * @length: length for request918 * @final: true if final op919 *920 * Prepare SSS HASH block for processing bytes in DMA mode. If it is called921 * after previous updates, fill up IV words. For final, calculate and set922 * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH923 * length as 2^63 so it will be never reached and set to zero prelow and924 * prehigh.925 *926 * This function does not start DMA transfer.927 */928static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length,929 bool final)930{931 struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);932 u32 prelow, prehigh, low, high;933 u32 configflags, swapflags;934 u64 tmplen;935 936 configflags = ctx->engine | SSS_HASH_INIT_BIT;937 938 if (likely(ctx->digcnt)) {939 s5p_hash_write_ctx_iv(dd, ctx);940 configflags |= SSS_HASH_USER_IV_EN;941 }942 943 if (final) {944 /* number of bytes for last part */945 low = length;946 high = 0;947 /* total number of bits prev hashed */948 tmplen = ctx->digcnt * 8;949 prelow = (u32)tmplen;950 prehigh = (u32)(tmplen >> 32);951 } else {952 prelow = 0;953 prehigh = 0;954 low = 0;955 high = BIT(31);956 }957 958 swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO |959 SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY;960 961 s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low);962 s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high);963 s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow);964 s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh);965 966 s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags);967 s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags);968}969 970/**971 * s5p_hash_xmit_dma() - start DMA hash processing972 * @dd: secss device973 * @length: length for request974 * @final: true if final op975 *976 * Update digcnt here, as it is needed for finup/final op.977 */978static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length,979 bool final)980{981 struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);982 unsigned int cnt;983 984 cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);985 if (!cnt) {986 dev_err(dd->dev, "dma_map_sg error\n");987 ctx->error = true;988 return -EINVAL;989 }990 991 set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);992 dd->hash_sg_iter = ctx->sg;993 dd->hash_sg_cnt = cnt;994 s5p_hash_write_ctrl(dd, length, final);995 ctx->digcnt += length;996 ctx->total -= length;997 998 /* catch last interrupt */999 if (final)1000 set_bit(HASH_FLAGS_FINAL, &dd->hash_flags);1001 1002 s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */1003 1004 return -EINPROGRESS;1005}1006 1007/**1008 * s5p_hash_copy_sgs() - copy request's bytes into new buffer1009 * @ctx: request context1010 * @sg: source scatterlist request1011 * @new_len: number of bytes to process from sg1012 *1013 * Allocate new buffer, copy data for HASH into it. If there was xmit_buf1014 * filled, copy it first, then copy data from sg into it. Prepare one sgl[0]1015 * with allocated buffer.1016 *1017 * Set bit in dd->hash_flag so we can free it after irq ends processing.1018 */1019static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx,1020 struct scatterlist *sg, unsigned int new_len)1021{1022 unsigned int pages, len;1023 void *buf;1024 1025 len = new_len + ctx->bufcnt;1026 pages = get_order(len);1027 1028 buf = (void *)__get_free_pages(GFP_ATOMIC, pages);1029 if (!buf) {1030 dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n");1031 ctx->error = true;1032 return -ENOMEM;1033 }1034 1035 if (ctx->bufcnt)1036 memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);1037 1038 scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip,1039 new_len, 0);1040 sg_init_table(ctx->sgl, 1);1041 sg_set_buf(ctx->sgl, buf, len);1042 ctx->sg = ctx->sgl;1043 ctx->sg_len = 1;1044 ctx->bufcnt = 0;1045 ctx->skip = 0;1046 set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags);1047 1048 return 0;1049}1050 1051/**1052 * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy1053 * @ctx: request context1054 * @sg: source scatterlist request1055 * @new_len: number of bytes to process from sg1056 *1057 * Allocate new scatterlist table, copy data for HASH into it. If there was1058 * xmit_buf filled, prepare it first, then copy page, length and offset from1059 * source sg into it, adjusting begin and/or end for skip offset and1060 * hash_later value.1061 *1062 * Resulting sg table will be assigned to ctx->sg. Set flag so we can free1063 * it after irq ends processing.1064 */1065static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx,1066 struct scatterlist *sg, unsigned int new_len)1067{1068 unsigned int skip = ctx->skip, n = sg_nents(sg);1069 struct scatterlist *tmp;1070 unsigned int len;1071 1072 if (ctx->bufcnt)1073 n++;1074 1075 ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);1076 if (!ctx->sg) {1077 ctx->error = true;1078 return -ENOMEM;1079 }1080 1081 sg_init_table(ctx->sg, n);1082 1083 tmp = ctx->sg;1084 1085 ctx->sg_len = 0;1086 1087 if (ctx->bufcnt) {1088 sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);1089 tmp = sg_next(tmp);1090 ctx->sg_len++;1091 }1092 1093 while (sg && skip >= sg->length) {1094 skip -= sg->length;1095 sg = sg_next(sg);1096 }1097 1098 while (sg && new_len) {1099 len = sg->length - skip;1100 if (new_len < len)1101 len = new_len;1102 1103 new_len -= len;1104 sg_set_page(tmp, sg_page(sg), len, sg->offset + skip);1105 skip = 0;1106 if (new_len <= 0)1107 sg_mark_end(tmp);1108 1109 tmp = sg_next(tmp);1110 ctx->sg_len++;1111 sg = sg_next(sg);1112 }1113 1114 set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags);1115 1116 return 0;1117}1118 1119/**1120 * s5p_hash_prepare_sgs() - prepare sg for processing1121 * @ctx: request context1122 * @sg: source scatterlist request1123 * @new_len: number of bytes to process from sg1124 * @final: final flag1125 *1126 * Check two conditions: (1) if buffers in sg have len aligned data, and (2)1127 * sg table have good aligned elements (list_ok). If one of this checks fails,1128 * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy1129 * data into this buffer and prepare request in sgl, or (2) allocates new sg1130 * table and prepare sg elements.1131 *1132 * For digest or finup all conditions can be good, and we may not need any1133 * fixes.1134 */1135static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx,1136 struct scatterlist *sg,1137 unsigned int new_len, bool final)1138{1139 unsigned int skip = ctx->skip, nbytes = new_len, n = 0;1140 bool aligned = true, list_ok = true;1141 struct scatterlist *sg_tmp = sg;1142 1143 if (!sg || !sg->length || !new_len)1144 return 0;1145 1146 if (skip || !final)1147 list_ok = false;1148 1149 while (nbytes > 0 && sg_tmp) {1150 n++;1151 if (skip >= sg_tmp->length) {1152 skip -= sg_tmp->length;1153 if (!sg_tmp->length) {1154 aligned = false;1155 break;1156 }1157 } else {1158 if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) {1159 aligned = false;1160 break;1161 }1162 1163 if (nbytes < sg_tmp->length - skip) {1164 list_ok = false;1165 break;1166 }1167 1168 nbytes -= sg_tmp->length - skip;1169 skip = 0;1170 }1171 1172 sg_tmp = sg_next(sg_tmp);1173 }1174 1175 if (!aligned)1176 return s5p_hash_copy_sgs(ctx, sg, new_len);1177 else if (!list_ok)1178 return s5p_hash_copy_sg_lists(ctx, sg, new_len);1179 1180 /*1181 * Have aligned data from previous operation and/or current1182 * Note: will enter here only if (digest or finup) and aligned1183 */1184 if (ctx->bufcnt) {1185 ctx->sg_len = n;1186 sg_init_table(ctx->sgl, 2);1187 sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt);1188 sg_chain(ctx->sgl, 2, sg);1189 ctx->sg = ctx->sgl;1190 ctx->sg_len++;1191 } else {1192 ctx->sg = sg;1193 ctx->sg_len = n;1194 }1195 1196 return 0;1197}1198 1199/**1200 * s5p_hash_prepare_request() - prepare request for processing1201 * @req: AHASH request1202 * @update: true if UPDATE op1203 *1204 * Note 1: we can have update flag _and_ final flag at the same time.1205 * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or1206 * either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or1207 * we have final op1208 */1209static int s5p_hash_prepare_request(struct ahash_request *req, bool update)1210{1211 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1212 bool final = ctx->finup;1213 int xmit_len, hash_later, nbytes;1214 int ret;1215 1216 if (update)1217 nbytes = req->nbytes;1218 else1219 nbytes = 0;1220 1221 ctx->total = nbytes + ctx->bufcnt;1222 if (!ctx->total)1223 return 0;1224 1225 if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) {1226 /* bytes left from previous request, so fill up to BUFLEN */1227 int len = BUFLEN - ctx->bufcnt % BUFLEN;1228 1229 if (len > nbytes)1230 len = nbytes;1231 1232 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,1233 0, len, 0);1234 ctx->bufcnt += len;1235 nbytes -= len;1236 ctx->skip = len;1237 } else {1238 ctx->skip = 0;1239 }1240 1241 if (ctx->bufcnt)1242 memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt);1243 1244 xmit_len = ctx->total;1245 if (final) {1246 hash_later = 0;1247 } else {1248 if (IS_ALIGNED(xmit_len, BUFLEN))1249 xmit_len -= BUFLEN;1250 else1251 xmit_len -= xmit_len & (BUFLEN - 1);1252 1253 hash_later = ctx->total - xmit_len;1254 /* copy hash_later bytes from end of req->src */1255 /* previous bytes are in xmit_buf, so no overwrite */1256 scatterwalk_map_and_copy(ctx->buffer, req->src,1257 req->nbytes - hash_later,1258 hash_later, 0);1259 }1260 1261 if (xmit_len > BUFLEN) {1262 ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later,1263 final);1264 if (ret)1265 return ret;1266 } else {1267 /* have buffered data only */1268 if (unlikely(!ctx->bufcnt)) {1269 /* first update didn't fill up buffer */1270 scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src,1271 0, xmit_len, 0);1272 }1273 1274 sg_init_table(ctx->sgl, 1);1275 sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len);1276 1277 ctx->sg = ctx->sgl;1278 ctx->sg_len = 1;1279 }1280 1281 ctx->bufcnt = hash_later;1282 if (!final)1283 ctx->total = xmit_len;1284 1285 return 0;1286}1287 1288/**1289 * s5p_hash_update_dma_stop() - unmap DMA1290 * @dd: secss device1291 *1292 * Unmap scatterlist ctx->sg.1293 */1294static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd)1295{1296 const struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);1297 1298 dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);1299 clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);1300}1301 1302/**1303 * s5p_hash_finish() - copy calculated digest to crypto layer1304 * @req: AHASH request1305 */1306static void s5p_hash_finish(struct ahash_request *req)1307{1308 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1309 struct s5p_aes_dev *dd = ctx->dd;1310 1311 if (ctx->digcnt)1312 s5p_hash_copy_result(req);1313 1314 dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt);1315}1316 1317/**1318 * s5p_hash_finish_req() - finish request1319 * @req: AHASH request1320 * @err: error1321 */1322static void s5p_hash_finish_req(struct ahash_request *req, int err)1323{1324 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1325 struct s5p_aes_dev *dd = ctx->dd;1326 unsigned long flags;1327 1328 if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags))1329 free_pages((unsigned long)sg_virt(ctx->sg),1330 get_order(ctx->sg->length));1331 1332 if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags))1333 kfree(ctx->sg);1334 1335 ctx->sg = NULL;1336 dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) |1337 BIT(HASH_FLAGS_SGS_COPIED));1338 1339 if (!err && !ctx->error) {1340 s5p_hash_read_msg(req);1341 if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags))1342 s5p_hash_finish(req);1343 } else {1344 ctx->error = true;1345 }1346 1347 spin_lock_irqsave(&dd->hash_lock, flags);1348 dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) |1349 BIT(HASH_FLAGS_DMA_READY) |1350 BIT(HASH_FLAGS_OUTPUT_READY));1351 spin_unlock_irqrestore(&dd->hash_lock, flags);1352 1353 if (req->base.complete)1354 ahash_request_complete(req, err);1355}1356 1357/**1358 * s5p_hash_handle_queue() - handle hash queue1359 * @dd: device s5p_aes_dev1360 * @req: AHASH request1361 *1362 * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the1363 * device then processes the first request from the dd->queue1364 *1365 * Returns: see s5p_hash_final below.1366 */1367static int s5p_hash_handle_queue(struct s5p_aes_dev *dd,1368 struct ahash_request *req)1369{1370 struct crypto_async_request *async_req, *backlog;1371 struct s5p_hash_reqctx *ctx;1372 unsigned long flags;1373 int err = 0, ret = 0;1374 1375retry:1376 spin_lock_irqsave(&dd->hash_lock, flags);1377 if (req)1378 ret = ahash_enqueue_request(&dd->hash_queue, req);1379 1380 if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {1381 spin_unlock_irqrestore(&dd->hash_lock, flags);1382 return ret;1383 }1384 1385 backlog = crypto_get_backlog(&dd->hash_queue);1386 async_req = crypto_dequeue_request(&dd->hash_queue);1387 if (async_req)1388 set_bit(HASH_FLAGS_BUSY, &dd->hash_flags);1389 1390 spin_unlock_irqrestore(&dd->hash_lock, flags);1391 1392 if (!async_req)1393 return ret;1394 1395 if (backlog)1396 crypto_request_complete(backlog, -EINPROGRESS);1397 1398 req = ahash_request_cast(async_req);1399 dd->hash_req = req;1400 ctx = ahash_request_ctx(req);1401 1402 err = s5p_hash_prepare_request(req, ctx->op_update);1403 if (err || !ctx->total)1404 goto out;1405 1406 dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n",1407 ctx->op_update, req->nbytes);1408 1409 s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT);1410 if (ctx->digcnt)1411 s5p_hash_write_iv(req); /* restore hash IV */1412 1413 if (ctx->op_update) { /* HASH_OP_UPDATE */1414 err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup);1415 if (err != -EINPROGRESS && ctx->finup && !ctx->error)1416 /* no final() after finup() */1417 err = s5p_hash_xmit_dma(dd, ctx->total, true);1418 } else { /* HASH_OP_FINAL */1419 err = s5p_hash_xmit_dma(dd, ctx->total, true);1420 }1421out:1422 if (err != -EINPROGRESS) {1423 /* hash_tasklet_cb will not finish it, so do it here */1424 s5p_hash_finish_req(req, err);1425 req = NULL;1426 1427 /*1428 * Execute next request immediately if there is anything1429 * in queue.1430 */1431 goto retry;1432 }1433 1434 return ret;1435}1436 1437/**1438 * s5p_hash_tasklet_cb() - hash tasklet1439 * @data: ptr to s5p_aes_dev1440 */1441static void s5p_hash_tasklet_cb(unsigned long data)1442{1443 struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data;1444 1445 if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {1446 s5p_hash_handle_queue(dd, NULL);1447 return;1448 }1449 1450 if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) {1451 if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE,1452 &dd->hash_flags)) {1453 s5p_hash_update_dma_stop(dd);1454 }1455 1456 if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY,1457 &dd->hash_flags)) {1458 /* hash or semi-hash ready */1459 clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags);1460 goto finish;1461 }1462 }1463 1464 return;1465 1466finish:1467 /* finish curent request */1468 s5p_hash_finish_req(dd->hash_req, 0);1469 1470 /* If we are not busy, process next req */1471 if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags))1472 s5p_hash_handle_queue(dd, NULL);1473}1474 1475/**1476 * s5p_hash_enqueue() - enqueue request1477 * @req: AHASH request1478 * @op: operation UPDATE (true) or FINAL (false)1479 *1480 * Returns: see s5p_hash_final below.1481 */1482static int s5p_hash_enqueue(struct ahash_request *req, bool op)1483{1484 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1485 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);1486 1487 ctx->op_update = op;1488 1489 return s5p_hash_handle_queue(tctx->dd, req);1490}1491 1492/**1493 * s5p_hash_update() - process the hash input data1494 * @req: AHASH request1495 *1496 * If request will fit in buffer, copy it and return immediately1497 * else enqueue it with OP_UPDATE.1498 *1499 * Returns: see s5p_hash_final below.1500 */1501static int s5p_hash_update(struct ahash_request *req)1502{1503 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1504 1505 if (!req->nbytes)1506 return 0;1507 1508 if (ctx->bufcnt + req->nbytes <= BUFLEN) {1509 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,1510 0, req->nbytes, 0);1511 ctx->bufcnt += req->nbytes;1512 return 0;1513 }1514 1515 return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */1516}1517 1518/**1519 * s5p_hash_final() - close up hash and calculate digest1520 * @req: AHASH request1521 *1522 * Note: in final req->src do not have any data, and req->nbytes can be1523 * non-zero.1524 *1525 * If there were no input data processed yet and the buffered hash data is1526 * less than BUFLEN (64) then calculate the final hash immediately by using1527 * SW algorithm fallback.1528 *1529 * Otherwise enqueues the current AHASH request with OP_FINAL operation op1530 * and finalize hash message in HW. Note that if digcnt!=0 then there were1531 * previous update op, so there are always some buffered bytes in ctx->buffer,1532 * which means that ctx->bufcnt!=01533 *1534 * Returns:1535 * 0 if the request has been processed immediately,1536 * -EINPROGRESS if the operation has been queued for later execution or is set1537 * to processing by HW,1538 * -EBUSY if queue is full and request should be resubmitted later,1539 * other negative values denotes an error.1540 */1541static int s5p_hash_final(struct ahash_request *req)1542{1543 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1544 1545 ctx->finup = true;1546 if (ctx->error)1547 return -EINVAL; /* uncompleted hash is not needed */1548 1549 if (!ctx->digcnt && ctx->bufcnt < BUFLEN) {1550 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);1551 1552 return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer,1553 ctx->bufcnt, req->result);1554 }1555 1556 return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */1557}1558 1559/**1560 * s5p_hash_finup() - process last req->src and calculate digest1561 * @req: AHASH request containing the last update data1562 *1563 * Return values: see s5p_hash_final above.1564 */1565static int s5p_hash_finup(struct ahash_request *req)1566{1567 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1568 int err1, err2;1569 1570 ctx->finup = true;1571 1572 err1 = s5p_hash_update(req);1573 if (err1 == -EINPROGRESS || err1 == -EBUSY)1574 return err1;1575 1576 /*1577 * final() has to be always called to cleanup resources even if1578 * update() failed, except EINPROGRESS or calculate digest for small1579 * size1580 */1581 err2 = s5p_hash_final(req);1582 1583 return err1 ?: err2;1584}1585 1586/**1587 * s5p_hash_init() - initialize AHASH request contex1588 * @req: AHASH request1589 *1590 * Init async hash request context.1591 */1592static int s5p_hash_init(struct ahash_request *req)1593{1594 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1595 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1596 struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);1597 1598 ctx->dd = tctx->dd;1599 ctx->error = false;1600 ctx->finup = false;1601 ctx->bufcnt = 0;1602 ctx->digcnt = 0;1603 ctx->total = 0;1604 ctx->skip = 0;1605 1606 dev_dbg(tctx->dd->dev, "init: digest size: %d\n",1607 crypto_ahash_digestsize(tfm));1608 1609 switch (crypto_ahash_digestsize(tfm)) {1610 case MD5_DIGEST_SIZE:1611 ctx->engine = SSS_HASH_ENGINE_MD5;1612 ctx->nregs = HASH_MD5_MAX_REG;1613 break;1614 case SHA1_DIGEST_SIZE:1615 ctx->engine = SSS_HASH_ENGINE_SHA1;1616 ctx->nregs = HASH_SHA1_MAX_REG;1617 break;1618 case SHA256_DIGEST_SIZE:1619 ctx->engine = SSS_HASH_ENGINE_SHA256;1620 ctx->nregs = HASH_SHA256_MAX_REG;1621 break;1622 default:1623 ctx->error = true;1624 return -EINVAL;1625 }1626 1627 return 0;1628}1629 1630/**1631 * s5p_hash_digest - calculate digest from req->src1632 * @req: AHASH request1633 *1634 * Return values: see s5p_hash_final above.1635 */1636static int s5p_hash_digest(struct ahash_request *req)1637{1638 return s5p_hash_init(req) ?: s5p_hash_finup(req);1639}1640 1641/**1642 * s5p_hash_cra_init_alg - init crypto alg transformation1643 * @tfm: crypto transformation1644 */1645static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm)1646{1647 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);1648 const char *alg_name = crypto_tfm_alg_name(tfm);1649 1650 tctx->dd = s5p_dev;1651 /* Allocate a fallback and abort if it failed. */1652 tctx->fallback = crypto_alloc_shash(alg_name, 0,1653 CRYPTO_ALG_NEED_FALLBACK);1654 if (IS_ERR(tctx->fallback)) {1655 pr_err("fallback alloc fails for '%s'\n", alg_name);1656 return PTR_ERR(tctx->fallback);1657 }1658 1659 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),1660 sizeof(struct s5p_hash_reqctx) + BUFLEN);1661 1662 return 0;1663}1664 1665/**1666 * s5p_hash_cra_init - init crypto tfm1667 * @tfm: crypto transformation1668 */1669static int s5p_hash_cra_init(struct crypto_tfm *tfm)1670{1671 return s5p_hash_cra_init_alg(tfm);1672}1673 1674/**1675 * s5p_hash_cra_exit - exit crypto tfm1676 * @tfm: crypto transformation1677 *1678 * free allocated fallback1679 */1680static void s5p_hash_cra_exit(struct crypto_tfm *tfm)1681{1682 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);1683 1684 crypto_free_shash(tctx->fallback);1685 tctx->fallback = NULL;1686}1687 1688/**1689 * s5p_hash_export - export hash state1690 * @req: AHASH request1691 * @out: buffer for exported state1692 */1693static int s5p_hash_export(struct ahash_request *req, void *out)1694{1695 const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1696 1697 memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt);1698 1699 return 0;1700}1701 1702/**1703 * s5p_hash_import - import hash state1704 * @req: AHASH request1705 * @in: buffer with state to be imported from1706 */1707static int s5p_hash_import(struct ahash_request *req, const void *in)1708{1709 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);1710 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1711 struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);1712 const struct s5p_hash_reqctx *ctx_in = in;1713 1714 memcpy(ctx, in, sizeof(*ctx) + BUFLEN);1715 if (ctx_in->bufcnt > BUFLEN) {1716 ctx->error = true;1717 return -EINVAL;1718 }1719 1720 ctx->dd = tctx->dd;1721 ctx->error = false;1722 1723 return 0;1724}1725 1726static struct ahash_alg algs_sha1_md5_sha256[] = {1727{1728 .init = s5p_hash_init,1729 .update = s5p_hash_update,1730 .final = s5p_hash_final,1731 .finup = s5p_hash_finup,1732 .digest = s5p_hash_digest,1733 .export = s5p_hash_export,1734 .import = s5p_hash_import,1735 .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,1736 .halg.digestsize = SHA1_DIGEST_SIZE,1737 .halg.base = {1738 .cra_name = "sha1",1739 .cra_driver_name = "exynos-sha1",1740 .cra_priority = 100,1741 .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |1742 CRYPTO_ALG_ASYNC |1743 CRYPTO_ALG_NEED_FALLBACK,1744 .cra_blocksize = HASH_BLOCK_SIZE,1745 .cra_ctxsize = sizeof(struct s5p_hash_ctx),1746 .cra_module = THIS_MODULE,1747 .cra_init = s5p_hash_cra_init,1748 .cra_exit = s5p_hash_cra_exit,1749 }1750},1751{1752 .init = s5p_hash_init,1753 .update = s5p_hash_update,1754 .final = s5p_hash_final,1755 .finup = s5p_hash_finup,1756 .digest = s5p_hash_digest,1757 .export = s5p_hash_export,1758 .import = s5p_hash_import,1759 .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,1760 .halg.digestsize = MD5_DIGEST_SIZE,1761 .halg.base = {1762 .cra_name = "md5",1763 .cra_driver_name = "exynos-md5",1764 .cra_priority = 100,1765 .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |1766 CRYPTO_ALG_ASYNC |1767 CRYPTO_ALG_NEED_FALLBACK,1768 .cra_blocksize = HASH_BLOCK_SIZE,1769 .cra_ctxsize = sizeof(struct s5p_hash_ctx),1770 .cra_module = THIS_MODULE,1771 .cra_init = s5p_hash_cra_init,1772 .cra_exit = s5p_hash_cra_exit,1773 }1774},1775{1776 .init = s5p_hash_init,1777 .update = s5p_hash_update,1778 .final = s5p_hash_final,1779 .finup = s5p_hash_finup,1780 .digest = s5p_hash_digest,1781 .export = s5p_hash_export,1782 .import = s5p_hash_import,1783 .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,1784 .halg.digestsize = SHA256_DIGEST_SIZE,1785 .halg.base = {1786 .cra_name = "sha256",1787 .cra_driver_name = "exynos-sha256",1788 .cra_priority = 100,1789 .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |1790 CRYPTO_ALG_ASYNC |1791 CRYPTO_ALG_NEED_FALLBACK,1792 .cra_blocksize = HASH_BLOCK_SIZE,1793 .cra_ctxsize = sizeof(struct s5p_hash_ctx),1794 .cra_module = THIS_MODULE,1795 .cra_init = s5p_hash_cra_init,1796 .cra_exit = s5p_hash_cra_exit,1797 }1798}1799 1800};1801 1802static void s5p_set_aes(struct s5p_aes_dev *dev,1803 const u8 *key, const u8 *iv, const u8 *ctr,1804 unsigned int keylen)1805{1806 void __iomem *keystart;1807 1808 if (iv)1809 memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv,1810 AES_BLOCK_SIZE);1811 1812 if (ctr)1813 memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), ctr,1814 AES_BLOCK_SIZE);1815 1816 if (keylen == AES_KEYSIZE_256)1817 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0);1818 else if (keylen == AES_KEYSIZE_192)1819 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2);1820 else1821 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4);1822 1823 memcpy_toio(keystart, key, keylen);1824}1825 1826static bool s5p_is_sg_aligned(struct scatterlist *sg)1827{1828 while (sg) {1829 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))1830 return false;1831 sg = sg_next(sg);1832 }1833 1834 return true;1835}1836 1837static int s5p_set_indata_start(struct s5p_aes_dev *dev,1838 struct skcipher_request *req)1839{1840 struct scatterlist *sg;1841 int err;1842 1843 dev->sg_src_cpy = NULL;1844 sg = req->src;1845 if (!s5p_is_sg_aligned(sg)) {1846 dev_dbg(dev->dev,1847 "At least one unaligned source scatter list, making a copy\n");1848 err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy);1849 if (err)1850 return err;1851 1852 sg = dev->sg_src_cpy;1853 }1854 1855 err = s5p_set_indata(dev, sg);1856 if (err) {1857 s5p_free_sg_cpy(dev, &dev->sg_src_cpy);1858 return err;1859 }1860 1861 return 0;1862}1863 1864static int s5p_set_outdata_start(struct s5p_aes_dev *dev,1865 struct skcipher_request *req)1866{1867 struct scatterlist *sg;1868 int err;1869 1870 dev->sg_dst_cpy = NULL;1871 sg = req->dst;1872 if (!s5p_is_sg_aligned(sg)) {1873 dev_dbg(dev->dev,1874 "At least one unaligned dest scatter list, making a copy\n");1875 err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy);1876 if (err)1877 return err;1878 1879 sg = dev->sg_dst_cpy;1880 }1881 1882 err = s5p_set_outdata(dev, sg);1883 if (err) {1884 s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);1885 return err;1886 }1887 1888 return 0;1889}1890 1891static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)1892{1893 struct skcipher_request *req = dev->req;1894 u32 aes_control;1895 unsigned long flags;1896 int err;1897 u8 *iv, *ctr;1898 1899 /* This sets bit [13:12] to 00, which selects 128-bit counter */1900 aes_control = SSS_AES_KEY_CHANGE_MODE;1901 if (mode & FLAGS_AES_DECRYPT)1902 aes_control |= SSS_AES_MODE_DECRYPT;1903 1904 if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {1905 aes_control |= SSS_AES_CHAIN_MODE_CBC;1906 iv = req->iv;1907 ctr = NULL;1908 } else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {1909 aes_control |= SSS_AES_CHAIN_MODE_CTR;1910 iv = NULL;1911 ctr = req->iv;1912 } else {1913 iv = NULL; /* AES_ECB */1914 ctr = NULL;1915 }1916 1917 if (dev->ctx->keylen == AES_KEYSIZE_192)1918 aes_control |= SSS_AES_KEY_SIZE_192;1919 else if (dev->ctx->keylen == AES_KEYSIZE_256)1920 aes_control |= SSS_AES_KEY_SIZE_256;1921 1922 aes_control |= SSS_AES_FIFO_MODE;1923 1924 /* as a variant it is possible to use byte swapping on DMA side */1925 aes_control |= SSS_AES_BYTESWAP_DI1926 | SSS_AES_BYTESWAP_DO1927 | SSS_AES_BYTESWAP_IV1928 | SSS_AES_BYTESWAP_KEY1929 | SSS_AES_BYTESWAP_CNT;1930 1931 spin_lock_irqsave(&dev->lock, flags);1932 1933 SSS_WRITE(dev, FCINTENCLR,1934 SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR);1935 SSS_WRITE(dev, FCFIFOCTRL, 0x00);1936 1937 err = s5p_set_indata_start(dev, req);1938 if (err)1939 goto indata_error;1940 1941 err = s5p_set_outdata_start(dev, req);1942 if (err)1943 goto outdata_error;1944 1945 SSS_AES_WRITE(dev, AES_CONTROL, aes_control);1946 s5p_set_aes(dev, dev->ctx->aes_key, iv, ctr, dev->ctx->keylen);1947 1948 s5p_set_dma_indata(dev, dev->sg_src);1949 s5p_set_dma_outdata(dev, dev->sg_dst);1950 1951 SSS_WRITE(dev, FCINTENSET,1952 SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET);1953 1954 spin_unlock_irqrestore(&dev->lock, flags);1955 1956 return;1957 1958outdata_error:1959 s5p_unset_indata(dev);1960 1961indata_error:1962 s5p_sg_done(dev);1963 dev->busy = false;1964 spin_unlock_irqrestore(&dev->lock, flags);1965 s5p_aes_complete(req, err);1966}1967 1968static void s5p_tasklet_cb(unsigned long data)1969{1970 struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data;1971 struct crypto_async_request *async_req, *backlog;1972 struct s5p_aes_reqctx *reqctx;1973 unsigned long flags;1974 1975 spin_lock_irqsave(&dev->lock, flags);1976 backlog = crypto_get_backlog(&dev->queue);1977 async_req = crypto_dequeue_request(&dev->queue);1978 1979 if (!async_req) {1980 dev->busy = false;1981 spin_unlock_irqrestore(&dev->lock, flags);1982 return;1983 }1984 spin_unlock_irqrestore(&dev->lock, flags);1985 1986 if (backlog)1987 crypto_request_complete(backlog, -EINPROGRESS);1988 1989 dev->req = skcipher_request_cast(async_req);1990 dev->ctx = crypto_tfm_ctx(dev->req->base.tfm);1991 reqctx = skcipher_request_ctx(dev->req);1992 1993 s5p_aes_crypt_start(dev, reqctx->mode);1994}1995 1996static int s5p_aes_handle_req(struct s5p_aes_dev *dev,1997 struct skcipher_request *req)1998{1999 unsigned long flags;2000 int err;2001 2002 spin_lock_irqsave(&dev->lock, flags);2003 err = crypto_enqueue_request(&dev->queue, &req->base);2004 if (dev->busy) {2005 spin_unlock_irqrestore(&dev->lock, flags);2006 return err;2007 }2008 dev->busy = true;2009 2010 spin_unlock_irqrestore(&dev->lock, flags);2011 2012 tasklet_schedule(&dev->tasklet);2013 2014 return err;2015}2016 2017static int s5p_aes_crypt(struct skcipher_request *req, unsigned long mode)2018{2019 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);2020 struct s5p_aes_reqctx *reqctx = skcipher_request_ctx(req);2021 struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm);2022 struct s5p_aes_dev *dev = ctx->dev;2023 2024 if (!req->cryptlen)2025 return 0;2026 2027 if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE) &&2028 ((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) {2029 dev_dbg(dev->dev, "request size is not exact amount of AES blocks\n");2030 return -EINVAL;2031 }2032 2033 reqctx->mode = mode;2034 2035 return s5p_aes_handle_req(dev, req);2036}2037 2038static int s5p_aes_setkey(struct crypto_skcipher *cipher,2039 const u8 *key, unsigned int keylen)2040{2041 struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);2042 struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);2043 2044 if (keylen != AES_KEYSIZE_128 &&2045 keylen != AES_KEYSIZE_192 &&2046 keylen != AES_KEYSIZE_256)2047 return -EINVAL;2048 2049 memcpy(ctx->aes_key, key, keylen);2050 ctx->keylen = keylen;2051 2052 return 0;2053}2054 2055static int s5p_aes_ecb_encrypt(struct skcipher_request *req)2056{2057 return s5p_aes_crypt(req, 0);2058}2059 2060static int s5p_aes_ecb_decrypt(struct skcipher_request *req)2061{2062 return s5p_aes_crypt(req, FLAGS_AES_DECRYPT);2063}2064 2065static int s5p_aes_cbc_encrypt(struct skcipher_request *req)2066{2067 return s5p_aes_crypt(req, FLAGS_AES_CBC);2068}2069 2070static int s5p_aes_cbc_decrypt(struct skcipher_request *req)2071{2072 return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);2073}2074 2075static int s5p_aes_ctr_crypt(struct skcipher_request *req)2076{2077 return s5p_aes_crypt(req, FLAGS_AES_CTR);2078}2079 2080static int s5p_aes_init_tfm(struct crypto_skcipher *tfm)2081{2082 struct s5p_aes_ctx *ctx = crypto_skcipher_ctx(tfm);2083 2084 ctx->dev = s5p_dev;2085 crypto_skcipher_set_reqsize(tfm, sizeof(struct s5p_aes_reqctx));2086 2087 return 0;2088}2089 2090static struct skcipher_alg algs[] = {2091 {2092 .base.cra_name = "ecb(aes)",2093 .base.cra_driver_name = "ecb-aes-s5p",2094 .base.cra_priority = 100,2095 .base.cra_flags = CRYPTO_ALG_ASYNC |2096 CRYPTO_ALG_KERN_DRIVER_ONLY,2097 .base.cra_blocksize = AES_BLOCK_SIZE,2098 .base.cra_ctxsize = sizeof(struct s5p_aes_ctx),2099 .base.cra_alignmask = 0x0f,2100 .base.cra_module = THIS_MODULE,2101 2102 .min_keysize = AES_MIN_KEY_SIZE,2103 .max_keysize = AES_MAX_KEY_SIZE,2104 .setkey = s5p_aes_setkey,2105 .encrypt = s5p_aes_ecb_encrypt,2106 .decrypt = s5p_aes_ecb_decrypt,2107 .init = s5p_aes_init_tfm,2108 },2109 {2110 .base.cra_name = "cbc(aes)",2111 .base.cra_driver_name = "cbc-aes-s5p",2112 .base.cra_priority = 100,2113 .base.cra_flags = CRYPTO_ALG_ASYNC |2114 CRYPTO_ALG_KERN_DRIVER_ONLY,2115 .base.cra_blocksize = AES_BLOCK_SIZE,2116 .base.cra_ctxsize = sizeof(struct s5p_aes_ctx),2117 .base.cra_alignmask = 0x0f,2118 .base.cra_module = THIS_MODULE,2119 2120 .min_keysize = AES_MIN_KEY_SIZE,2121 .max_keysize = AES_MAX_KEY_SIZE,2122 .ivsize = AES_BLOCK_SIZE,2123 .setkey = s5p_aes_setkey,2124 .encrypt = s5p_aes_cbc_encrypt,2125 .decrypt = s5p_aes_cbc_decrypt,2126 .init = s5p_aes_init_tfm,2127 },2128 {2129 .base.cra_name = "ctr(aes)",2130 .base.cra_driver_name = "ctr-aes-s5p",2131 .base.cra_priority = 100,2132 .base.cra_flags = CRYPTO_ALG_ASYNC |2133 CRYPTO_ALG_KERN_DRIVER_ONLY,2134 .base.cra_blocksize = 1,2135 .base.cra_ctxsize = sizeof(struct s5p_aes_ctx),2136 .base.cra_alignmask = 0x0f,2137 .base.cra_module = THIS_MODULE,2138 2139 .min_keysize = AES_MIN_KEY_SIZE,2140 .max_keysize = AES_MAX_KEY_SIZE,2141 .ivsize = AES_BLOCK_SIZE,2142 .setkey = s5p_aes_setkey,2143 .encrypt = s5p_aes_ctr_crypt,2144 .decrypt = s5p_aes_ctr_crypt,2145 .init = s5p_aes_init_tfm,2146 },2147};2148 2149static int s5p_aes_probe(struct platform_device *pdev)2150{2151 struct device *dev = &pdev->dev;2152 int i, j, err;2153 const struct samsung_aes_variant *variant;2154 struct s5p_aes_dev *pdata;2155 struct resource *res;2156 unsigned int hash_i;2157 2158 if (s5p_dev)2159 return -EEXIST;2160 2161 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);2162 if (!pdata)2163 return -ENOMEM;2164 2165 variant = find_s5p_sss_version(pdev);2166 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);2167 if (!res)2168 return -EINVAL;2169 2170 /*2171 * Note: HASH and PRNG uses the same registers in secss, avoid2172 * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG2173 * is enabled in config. We need larger size for HASH registers in2174 * secss, current describe only AES/DES2175 */2176 if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) {2177 if (variant == &exynos_aes_data) {2178 res->end += 0x300;2179 pdata->use_hash = true;2180 }2181 }2182 2183 pdata->res = res;2184 pdata->ioaddr = devm_ioremap_resource(dev, res);2185 if (IS_ERR(pdata->ioaddr)) {2186 if (!pdata->use_hash)2187 return PTR_ERR(pdata->ioaddr);2188 /* try AES without HASH */2189 res->end -= 0x300;2190 pdata->use_hash = false;2191 pdata->ioaddr = devm_ioremap_resource(dev, res);2192 if (IS_ERR(pdata->ioaddr))2193 return PTR_ERR(pdata->ioaddr);2194 }2195 2196 pdata->clk = devm_clk_get(dev, variant->clk_names[0]);2197 if (IS_ERR(pdata->clk))2198 return dev_err_probe(dev, PTR_ERR(pdata->clk),2199 "failed to find secss clock %s\n",2200 variant->clk_names[0]);2201 2202 err = clk_prepare_enable(pdata->clk);2203 if (err < 0) {2204 dev_err(dev, "Enabling clock %s failed, err %d\n",2205 variant->clk_names[0], err);2206 return err;2207 }2208 2209 if (variant->clk_names[1]) {2210 pdata->pclk = devm_clk_get(dev, variant->clk_names[1]);2211 if (IS_ERR(pdata->pclk)) {2212 err = dev_err_probe(dev, PTR_ERR(pdata->pclk),2213 "failed to find clock %s\n",2214 variant->clk_names[1]);2215 goto err_clk;2216 }2217 2218 err = clk_prepare_enable(pdata->pclk);2219 if (err < 0) {2220 dev_err(dev, "Enabling clock %s failed, err %d\n",2221 variant->clk_names[0], err);2222 goto err_clk;2223 }2224 } else {2225 pdata->pclk = NULL;2226 }2227 2228 spin_lock_init(&pdata->lock);2229 spin_lock_init(&pdata->hash_lock);2230 2231 pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset;2232 pdata->io_hash_base = pdata->ioaddr + variant->hash_offset;2233 2234 pdata->irq_fc = platform_get_irq(pdev, 0);2235 if (pdata->irq_fc < 0) {2236 err = pdata->irq_fc;2237 dev_warn(dev, "feed control interrupt is not available.\n");2238 goto err_irq;2239 }2240 err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL,2241 s5p_aes_interrupt, IRQF_ONESHOT,2242 pdev->name, pdev);2243 if (err < 0) {2244 dev_warn(dev, "feed control interrupt is not available.\n");2245 goto err_irq;2246 }2247 2248 pdata->busy = false;2249 pdata->dev = dev;2250 platform_set_drvdata(pdev, pdata);2251 s5p_dev = pdata;2252 2253 tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata);2254 crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);2255 2256 for (i = 0; i < ARRAY_SIZE(algs); i++) {2257 err = crypto_register_skcipher(&algs[i]);2258 if (err)2259 goto err_algs;2260 }2261 2262 if (pdata->use_hash) {2263 tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb,2264 (unsigned long)pdata);2265 crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH);2266 2267 for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256);2268 hash_i++) {2269 struct ahash_alg *alg;2270 2271 alg = &algs_sha1_md5_sha256[hash_i];2272 err = crypto_register_ahash(alg);2273 if (err) {2274 dev_err(dev, "can't register '%s': %d\n",2275 alg->halg.base.cra_driver_name, err);2276 goto err_hash;2277 }2278 }2279 }2280 2281 dev_info(dev, "s5p-sss driver registered\n");2282 2283 return 0;2284 2285err_hash:2286 for (j = hash_i - 1; j >= 0; j--)2287 crypto_unregister_ahash(&algs_sha1_md5_sha256[j]);2288 2289 tasklet_kill(&pdata->hash_tasklet);2290 res->end -= 0x300;2291 2292err_algs:2293 if (i < ARRAY_SIZE(algs))2294 dev_err(dev, "can't register '%s': %d\n", algs[i].base.cra_name,2295 err);2296 2297 for (j = 0; j < i; j++)2298 crypto_unregister_skcipher(&algs[j]);2299 2300 tasklet_kill(&pdata->tasklet);2301 2302err_irq:2303 clk_disable_unprepare(pdata->pclk);2304 2305err_clk:2306 clk_disable_unprepare(pdata->clk);2307 s5p_dev = NULL;2308 2309 return err;2310}2311 2312static void s5p_aes_remove(struct platform_device *pdev)2313{2314 struct s5p_aes_dev *pdata = platform_get_drvdata(pdev);2315 int i;2316 2317 for (i = 0; i < ARRAY_SIZE(algs); i++)2318 crypto_unregister_skcipher(&algs[i]);2319 2320 tasklet_kill(&pdata->tasklet);2321 if (pdata->use_hash) {2322 for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--)2323 crypto_unregister_ahash(&algs_sha1_md5_sha256[i]);2324 2325 pdata->res->end -= 0x300;2326 tasklet_kill(&pdata->hash_tasklet);2327 pdata->use_hash = false;2328 }2329 2330 clk_disable_unprepare(pdata->pclk);2331 2332 clk_disable_unprepare(pdata->clk);2333 s5p_dev = NULL;2334}2335 2336static struct platform_driver s5p_aes_crypto = {2337 .probe = s5p_aes_probe,2338 .remove_new = s5p_aes_remove,2339 .driver = {2340 .name = "s5p-secss",2341 .of_match_table = s5p_sss_dt_match,2342 },2343};2344 2345module_platform_driver(s5p_aes_crypto);2346 2347MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");2348MODULE_LICENSE("GPL v2");2349MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");2350MODULE_AUTHOR("Kamil Konieczny <k.konieczny@partner.samsung.com>");2351