2787 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) STMicroelectronics SA 20174 * Author: Fabien Dessenne <fabien.dessenne@st.com>5 * Ux500 support taken from snippets in the old Ux500 cryp driver6 */7 8#include <crypto/aes.h>9#include <crypto/engine.h>10#include <crypto/internal/aead.h>11#include <crypto/internal/des.h>12#include <crypto/internal/skcipher.h>13#include <crypto/scatterwalk.h>14#include <linux/bottom_half.h>15#include <linux/clk.h>16#include <linux/delay.h>17#include <linux/dma-mapping.h>18#include <linux/dmaengine.h>19#include <linux/err.h>20#include <linux/iopoll.h>21#include <linux/interrupt.h>22#include <linux/kernel.h>23#include <linux/module.h>24#include <linux/of.h>25#include <linux/platform_device.h>26#include <linux/pm_runtime.h>27#include <linux/reset.h>28#include <linux/string.h>29 30#define DRIVER_NAME "stm32-cryp"31 32/* Bit [0] encrypt / decrypt */33#define FLG_ENCRYPT BIT(0)34/* Bit [8..1] algo & operation mode */35#define FLG_AES BIT(1)36#define FLG_DES BIT(2)37#define FLG_TDES BIT(3)38#define FLG_ECB BIT(4)39#define FLG_CBC BIT(5)40#define FLG_CTR BIT(6)41#define FLG_GCM BIT(7)42#define FLG_CCM BIT(8)43/* Mode mask = bits [15..0] */44#define FLG_MODE_MASK GENMASK(15, 0)45/* Bit [31..16] status */46#define FLG_IN_OUT_DMA BIT(16)47#define FLG_HEADER_DMA BIT(17)48 49/* Registers */50#define CRYP_CR 0x0000000051#define CRYP_SR 0x0000000452#define CRYP_DIN 0x0000000853#define CRYP_DOUT 0x0000000C54#define CRYP_DMACR 0x0000001055#define CRYP_IMSCR 0x0000001456#define CRYP_RISR 0x0000001857#define CRYP_MISR 0x0000001C58#define CRYP_K0LR 0x0000002059#define CRYP_K0RR 0x0000002460#define CRYP_K1LR 0x0000002861#define CRYP_K1RR 0x0000002C62#define CRYP_K2LR 0x0000003063#define CRYP_K2RR 0x0000003464#define CRYP_K3LR 0x0000003865#define CRYP_K3RR 0x0000003C66#define CRYP_IV0LR 0x0000004067#define CRYP_IV0RR 0x0000004468#define CRYP_IV1LR 0x0000004869#define CRYP_IV1RR 0x0000004C70#define CRYP_CSGCMCCM0R 0x0000005071#define CRYP_CSGCM0R 0x0000007072 73#define UX500_CRYP_CR 0x0000000074#define UX500_CRYP_SR 0x0000000475#define UX500_CRYP_DIN 0x0000000876#define UX500_CRYP_DINSIZE 0x0000000C77#define UX500_CRYP_DOUT 0x0000001078#define UX500_CRYP_DOUSIZE 0x0000001479#define UX500_CRYP_DMACR 0x0000001880#define UX500_CRYP_IMSC 0x0000001C81#define UX500_CRYP_RIS 0x0000002082#define UX500_CRYP_MIS 0x0000002483#define UX500_CRYP_K1L 0x0000002884#define UX500_CRYP_K1R 0x0000002C85#define UX500_CRYP_K2L 0x0000003086#define UX500_CRYP_K2R 0x0000003487#define UX500_CRYP_K3L 0x0000003888#define UX500_CRYP_K3R 0x0000003C89#define UX500_CRYP_K4L 0x0000004090#define UX500_CRYP_K4R 0x0000004491#define UX500_CRYP_IV0L 0x0000004892#define UX500_CRYP_IV0R 0x0000004C93#define UX500_CRYP_IV1L 0x0000005094#define UX500_CRYP_IV1R 0x0000005495 96/* Registers values */97#define CR_DEC_NOT_ENC 0x0000000498#define CR_TDES_ECB 0x0000000099#define CR_TDES_CBC 0x00000008100#define CR_DES_ECB 0x00000010101#define CR_DES_CBC 0x00000018102#define CR_AES_ECB 0x00000020103#define CR_AES_CBC 0x00000028104#define CR_AES_CTR 0x00000030105#define CR_AES_KP 0x00000038 /* Not on Ux500 */106#define CR_AES_XTS 0x00000038 /* Only on Ux500 */107#define CR_AES_GCM 0x00080000108#define CR_AES_CCM 0x00080008109#define CR_AES_UNKNOWN 0xFFFFFFFF110#define CR_ALGO_MASK 0x00080038111#define CR_DATA32 0x00000000112#define CR_DATA16 0x00000040113#define CR_DATA8 0x00000080114#define CR_DATA1 0x000000C0115#define CR_KEY128 0x00000000116#define CR_KEY192 0x00000100117#define CR_KEY256 0x00000200118#define CR_KEYRDEN 0x00000400 /* Only on Ux500 */119#define CR_KSE 0x00000800 /* Only on Ux500 */120#define CR_FFLUSH 0x00004000121#define CR_CRYPEN 0x00008000122#define CR_PH_INIT 0x00000000123#define CR_PH_HEADER 0x00010000124#define CR_PH_PAYLOAD 0x00020000125#define CR_PH_FINAL 0x00030000126#define CR_PH_MASK 0x00030000127#define CR_NBPBL_SHIFT 20128 129#define SR_IFNF BIT(1)130#define SR_OFNE BIT(2)131#define SR_BUSY BIT(8)132 133#define DMACR_DIEN BIT(0)134#define DMACR_DOEN BIT(1)135 136#define IMSCR_IN BIT(0)137#define IMSCR_OUT BIT(1)138 139#define MISR_IN BIT(0)140#define MISR_OUT BIT(1)141 142/* Misc */143#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))144#define GCM_CTR_INIT 2145#define CRYP_AUTOSUSPEND_DELAY 50146 147#define CRYP_DMA_BURST_REG 4148 149enum stm32_dma_mode {150 NO_DMA,151 DMA_PLAIN_SG,152 DMA_NEED_SG_TRUNC153};154 155struct stm32_cryp_caps {156 bool aeads_support;157 bool linear_aes_key;158 bool kp_mode;159 bool iv_protection;160 bool swap_final;161 bool padding_wa;162 u32 cr;163 u32 sr;164 u32 din;165 u32 dout;166 u32 dmacr;167 u32 imsc;168 u32 mis;169 u32 k1l;170 u32 k1r;171 u32 k3r;172 u32 iv0l;173 u32 iv0r;174 u32 iv1l;175 u32 iv1r;176};177 178struct stm32_cryp_ctx {179 struct stm32_cryp *cryp;180 int keylen;181 __be32 key[AES_KEYSIZE_256 / sizeof(u32)];182 unsigned long flags;183};184 185struct stm32_cryp_reqctx {186 unsigned long mode;187};188 189struct stm32_cryp {190 struct list_head list;191 struct device *dev;192 void __iomem *regs;193 phys_addr_t phys_base;194 struct clk *clk;195 unsigned long flags;196 u32 irq_status;197 const struct stm32_cryp_caps *caps;198 struct stm32_cryp_ctx *ctx;199 200 struct crypto_engine *engine;201 202 struct skcipher_request *req;203 struct aead_request *areq;204 205 size_t authsize;206 size_t hw_blocksize;207 208 size_t payload_in;209 size_t header_in;210 size_t payload_out;211 212 /* DMA process fields */213 struct scatterlist *in_sg;214 struct scatterlist *header_sg;215 struct scatterlist *out_sg;216 size_t in_sg_len;217 size_t header_sg_len;218 size_t out_sg_len;219 struct completion dma_completion;220 221 struct dma_chan *dma_lch_in;222 struct dma_chan *dma_lch_out;223 enum stm32_dma_mode dma_mode;224 225 /* IT process fields */226 struct scatter_walk in_walk;227 struct scatter_walk out_walk;228 229 __be32 last_ctr[4];230 u32 gcm_ctr;231};232 233struct stm32_cryp_list {234 struct list_head dev_list;235 spinlock_t lock; /* protect dev_list */236};237 238static struct stm32_cryp_list cryp_list = {239 .dev_list = LIST_HEAD_INIT(cryp_list.dev_list),240 .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock),241};242 243static inline bool is_aes(struct stm32_cryp *cryp)244{245 return cryp->flags & FLG_AES;246}247 248static inline bool is_des(struct stm32_cryp *cryp)249{250 return cryp->flags & FLG_DES;251}252 253static inline bool is_tdes(struct stm32_cryp *cryp)254{255 return cryp->flags & FLG_TDES;256}257 258static inline bool is_ecb(struct stm32_cryp *cryp)259{260 return cryp->flags & FLG_ECB;261}262 263static inline bool is_cbc(struct stm32_cryp *cryp)264{265 return cryp->flags & FLG_CBC;266}267 268static inline bool is_ctr(struct stm32_cryp *cryp)269{270 return cryp->flags & FLG_CTR;271}272 273static inline bool is_gcm(struct stm32_cryp *cryp)274{275 return cryp->flags & FLG_GCM;276}277 278static inline bool is_ccm(struct stm32_cryp *cryp)279{280 return cryp->flags & FLG_CCM;281}282 283static inline bool is_encrypt(struct stm32_cryp *cryp)284{285 return cryp->flags & FLG_ENCRYPT;286}287 288static inline bool is_decrypt(struct stm32_cryp *cryp)289{290 return !is_encrypt(cryp);291}292 293static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst)294{295 return readl_relaxed(cryp->regs + ofst);296}297 298static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val)299{300 writel_relaxed(val, cryp->regs + ofst);301}302 303static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)304{305 u32 status;306 307 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status,308 !(status & SR_BUSY), 10, 100000);309}310 311static inline void stm32_cryp_enable(struct stm32_cryp *cryp)312{313 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_CRYPEN,314 cryp->regs + cryp->caps->cr);315}316 317static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp)318{319 u32 status;320 321 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->cr, status,322 !(status & CR_CRYPEN), 10, 100000);323}324 325static inline int stm32_cryp_wait_input(struct stm32_cryp *cryp)326{327 u32 status;328 329 return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status,330 status & SR_IFNF, 1, 10);331}332 333static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp)334{335 u32 status;336 337 return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status,338 status & SR_OFNE, 1, 10);339}340 341static inline void stm32_cryp_key_read_enable(struct stm32_cryp *cryp)342{343 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_KEYRDEN,344 cryp->regs + cryp->caps->cr);345}346 347static inline void stm32_cryp_key_read_disable(struct stm32_cryp *cryp)348{349 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) & ~CR_KEYRDEN,350 cryp->regs + cryp->caps->cr);351}352 353static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp);354static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp);355static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp);356static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);357static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);358static int stm32_cryp_dma_start(struct stm32_cryp *cryp);359static int stm32_cryp_it_start(struct stm32_cryp *cryp);360 361static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)362{363 struct stm32_cryp *tmp, *cryp = NULL;364 365 spin_lock_bh(&cryp_list.lock);366 if (!ctx->cryp) {367 list_for_each_entry(tmp, &cryp_list.dev_list, list) {368 cryp = tmp;369 break;370 }371 ctx->cryp = cryp;372 } else {373 cryp = ctx->cryp;374 }375 376 spin_unlock_bh(&cryp_list.lock);377 378 return cryp;379}380 381static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv)382{383 if (!iv)384 return;385 386 stm32_cryp_write(cryp, cryp->caps->iv0l, be32_to_cpu(*iv++));387 stm32_cryp_write(cryp, cryp->caps->iv0r, be32_to_cpu(*iv++));388 389 if (is_aes(cryp)) {390 stm32_cryp_write(cryp, cryp->caps->iv1l, be32_to_cpu(*iv++));391 stm32_cryp_write(cryp, cryp->caps->iv1r, be32_to_cpu(*iv++));392 }393}394 395static void stm32_cryp_get_iv(struct stm32_cryp *cryp)396{397 struct skcipher_request *req = cryp->req;398 __be32 *tmp = (void *)req->iv;399 400 if (!tmp)401 return;402 403 if (cryp->caps->iv_protection)404 stm32_cryp_key_read_enable(cryp);405 406 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l));407 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r));408 409 if (is_aes(cryp)) {410 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l));411 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r));412 }413 414 if (cryp->caps->iv_protection)415 stm32_cryp_key_read_disable(cryp);416}417 418/**419 * ux500_swap_bits_in_byte() - mirror the bits in a byte420 * @b: the byte to be mirrored421 *422 * The bits are swapped the following way:423 * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and424 * nibble 2 (n2) bits 4-7.425 *426 * Nibble 1 (n1):427 * (The "old" (moved) bit is replaced with a zero)428 * 1. Move bit 6 and 7, 4 positions to the left.429 * 2. Move bit 3 and 5, 2 positions to the left.430 * 3. Move bit 1-4, 1 position to the left.431 *432 * Nibble 2 (n2):433 * 1. Move bit 0 and 1, 4 positions to the right.434 * 2. Move bit 2 and 4, 2 positions to the right.435 * 3. Move bit 3-6, 1 position to the right.436 *437 * Combine the two nibbles to a complete and swapped byte.438 */439static inline u8 ux500_swap_bits_in_byte(u8 b)440{441#define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */442#define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5,443 right shift 2 */444#define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4,445 right shift 1 */446#define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */447#define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4,448 left shift 2 */449#define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6,450 left shift 1 */451 452 u8 n1;453 u8 n2;454 455 /* Swap most significant nibble */456 /* Right shift 4, bits 6 and 7 */457 n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4));458 /* Right shift 2, bits 3 and 5 */459 n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2));460 /* Right shift 1, bits 1-4 */461 n1 = (n1 & R_SHIFT_1_MASK) >> 1;462 463 /* Swap least significant nibble */464 /* Left shift 4, bits 0 and 1 */465 n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4));466 /* Left shift 2, bits 2 and 4 */467 n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2));468 /* Left shift 1, bits 3-6 */469 n2 = (n2 & L_SHIFT_1_MASK) << 1;470 471 return n1 | n2;472}473 474/**475 * ux500_swizzle_key() - Shuffle around words and bits in the AES key476 * @in: key to swizzle477 * @out: swizzled key478 * @len: length of key, in bytes479 *480 * This "key swizzling procedure" is described in the examples in the481 * DB8500 design specification. There is no real description of why482 * the bits have been arranged like this in the hardware.483 */484static inline void ux500_swizzle_key(const u8 *in, u8 *out, u32 len)485{486 int i = 0;487 int bpw = sizeof(u32);488 int j;489 int index = 0;490 491 j = len - bpw;492 while (j >= 0) {493 for (i = 0; i < bpw; i++) {494 index = len - j - bpw + i;495 out[j + i] =496 ux500_swap_bits_in_byte(in[index]);497 }498 j -= bpw;499 }500}501 502static void stm32_cryp_hw_write_key(struct stm32_cryp *c)503{504 unsigned int i;505 int r_id;506 507 if (is_des(c)) {508 stm32_cryp_write(c, c->caps->k1l, be32_to_cpu(c->ctx->key[0]));509 stm32_cryp_write(c, c->caps->k1r, be32_to_cpu(c->ctx->key[1]));510 return;511 }512 513 /*514 * On the Ux500 the AES key is considered as a single bit sequence515 * of 128, 192 or 256 bits length. It is written linearly into the516 * registers from K1L and down, and need to be processed to become517 * a proper big-endian bit sequence.518 */519 if (is_aes(c) && c->caps->linear_aes_key) {520 u32 tmpkey[8];521 522 ux500_swizzle_key((u8 *)c->ctx->key,523 (u8 *)tmpkey, c->ctx->keylen);524 525 r_id = c->caps->k1l;526 for (i = 0; i < c->ctx->keylen / sizeof(u32); i++, r_id += 4)527 stm32_cryp_write(c, r_id, tmpkey[i]);528 529 return;530 }531 532 r_id = c->caps->k3r;533 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)534 stm32_cryp_write(c, r_id, be32_to_cpu(c->ctx->key[i - 1]));535}536 537static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp)538{539 if (is_aes(cryp) && is_ecb(cryp))540 return CR_AES_ECB;541 542 if (is_aes(cryp) && is_cbc(cryp))543 return CR_AES_CBC;544 545 if (is_aes(cryp) && is_ctr(cryp))546 return CR_AES_CTR;547 548 if (is_aes(cryp) && is_gcm(cryp))549 return CR_AES_GCM;550 551 if (is_aes(cryp) && is_ccm(cryp))552 return CR_AES_CCM;553 554 if (is_des(cryp) && is_ecb(cryp))555 return CR_DES_ECB;556 557 if (is_des(cryp) && is_cbc(cryp))558 return CR_DES_CBC;559 560 if (is_tdes(cryp) && is_ecb(cryp))561 return CR_TDES_ECB;562 563 if (is_tdes(cryp) && is_cbc(cryp))564 return CR_TDES_CBC;565 566 dev_err(cryp->dev, "Unknown mode\n");567 return CR_AES_UNKNOWN;568}569 570static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp)571{572 return is_encrypt(cryp) ? cryp->areq->cryptlen :573 cryp->areq->cryptlen - cryp->authsize;574}575 576static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)577{578 int ret;579 __be32 iv[4];580 581 /* Phase 1 : init */582 memcpy(iv, cryp->areq->iv, 12);583 iv[3] = cpu_to_be32(GCM_CTR_INIT);584 cryp->gcm_ctr = GCM_CTR_INIT;585 stm32_cryp_hw_write_iv(cryp, iv);586 587 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN);588 589 /* Wait for end of processing */590 ret = stm32_cryp_wait_enable(cryp);591 if (ret) {592 dev_err(cryp->dev, "Timeout (gcm init)\n");593 return ret;594 }595 596 /* Prepare next phase */597 if (cryp->areq->assoclen) {598 cfg |= CR_PH_HEADER;599 stm32_cryp_write(cryp, cryp->caps->cr, cfg);600 } else if (stm32_cryp_get_input_text_len(cryp)) {601 cfg |= CR_PH_PAYLOAD;602 stm32_cryp_write(cryp, cryp->caps->cr, cfg);603 }604 605 return 0;606}607 608static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp)609{610 u32 cfg;611 int err;612 613 /* Check if whole header written */614 if (!cryp->header_in) {615 /* Wait for completion */616 err = stm32_cryp_wait_busy(cryp);617 if (err) {618 dev_err(cryp->dev, "Timeout (gcm/ccm header)\n");619 stm32_cryp_write(cryp, cryp->caps->imsc, 0);620 stm32_cryp_finish_req(cryp, err);621 return;622 }623 624 if (stm32_cryp_get_input_text_len(cryp)) {625 /* Phase 3 : payload */626 cfg = stm32_cryp_read(cryp, cryp->caps->cr);627 cfg &= ~CR_CRYPEN;628 stm32_cryp_write(cryp, cryp->caps->cr, cfg);629 630 cfg &= ~CR_PH_MASK;631 cfg |= CR_PH_PAYLOAD | CR_CRYPEN;632 stm32_cryp_write(cryp, cryp->caps->cr, cfg);633 } else {634 /*635 * Phase 4 : tag.636 * Nothing to read, nothing to write, caller have to637 * end request638 */639 }640 }641}642 643static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp)644{645 size_t written;646 size_t len;647 u32 alen = cryp->areq->assoclen;648 u32 block[AES_BLOCK_32] = {0};649 u8 *b8 = (u8 *)block;650 651 if (alen <= 65280) {652 /* Write first u32 of B1 */653 b8[0] = (alen >> 8) & 0xFF;654 b8[1] = alen & 0xFF;655 len = 2;656 } else {657 /* Build the two first u32 of B1 */658 b8[0] = 0xFF;659 b8[1] = 0xFE;660 b8[2] = (alen & 0xFF000000) >> 24;661 b8[3] = (alen & 0x00FF0000) >> 16;662 b8[4] = (alen & 0x0000FF00) >> 8;663 b8[5] = alen & 0x000000FF;664 len = 6;665 }666 667 written = min_t(size_t, AES_BLOCK_SIZE - len, alen);668 669 scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0);670 671 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32);672 673 cryp->header_in -= written;674 675 stm32_crypt_gcmccm_end_header(cryp);676}677 678static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)679{680 int ret;681 u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32];682 u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32;683 __be32 *bd;684 u32 *d;685 unsigned int i, textlen;686 687 /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */688 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);689 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);690 iv[AES_BLOCK_SIZE - 1] = 1;691 stm32_cryp_hw_write_iv(cryp, (__be32 *)iv);692 693 /* Build B0 */694 memcpy(b0, iv, AES_BLOCK_SIZE);695 696 b0[0] |= (8 * ((cryp->authsize - 2) / 2));697 698 if (cryp->areq->assoclen)699 b0[0] |= 0x40;700 701 textlen = stm32_cryp_get_input_text_len(cryp);702 703 b0[AES_BLOCK_SIZE - 2] = textlen >> 8;704 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;705 706 /* Enable HW */707 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN);708 709 /* Write B0 */710 d = (u32 *)b0;711 bd = (__be32 *)b0;712 713 for (i = 0; i < AES_BLOCK_32; i++) {714 u32 xd = d[i];715 716 if (!cryp->caps->padding_wa)717 xd = be32_to_cpu(bd[i]);718 stm32_cryp_write(cryp, cryp->caps->din, xd);719 }720 721 /* Wait for end of processing */722 ret = stm32_cryp_wait_enable(cryp);723 if (ret) {724 dev_err(cryp->dev, "Timeout (ccm init)\n");725 return ret;726 }727 728 /* Prepare next phase */729 if (cryp->areq->assoclen) {730 cfg |= CR_PH_HEADER | CR_CRYPEN;731 stm32_cryp_write(cryp, cryp->caps->cr, cfg);732 733 /* Write first (special) block (may move to next phase [payload]) */734 stm32_cryp_write_ccm_first_header(cryp);735 } else if (stm32_cryp_get_input_text_len(cryp)) {736 cfg |= CR_PH_PAYLOAD;737 stm32_cryp_write(cryp, cryp->caps->cr, cfg);738 }739 740 return 0;741}742 743static int stm32_cryp_hw_init(struct stm32_cryp *cryp)744{745 int ret;746 u32 cfg, hw_mode;747 748 pm_runtime_get_sync(cryp->dev);749 750 /* Disable interrupt */751 stm32_cryp_write(cryp, cryp->caps->imsc, 0);752 753 /* Set configuration */754 cfg = CR_DATA8 | CR_FFLUSH;755 756 switch (cryp->ctx->keylen) {757 case AES_KEYSIZE_128:758 cfg |= CR_KEY128;759 break;760 761 case AES_KEYSIZE_192:762 cfg |= CR_KEY192;763 break;764 765 default:766 case AES_KEYSIZE_256:767 cfg |= CR_KEY256;768 break;769 }770 771 hw_mode = stm32_cryp_get_hw_mode(cryp);772 if (hw_mode == CR_AES_UNKNOWN)773 return -EINVAL;774 775 /* AES ECB/CBC decrypt: run key preparation first */776 if (is_decrypt(cryp) &&777 ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {778 /* Configure in key preparation mode */779 if (cryp->caps->kp_mode)780 stm32_cryp_write(cryp, cryp->caps->cr,781 cfg | CR_AES_KP);782 else783 stm32_cryp_write(cryp,784 cryp->caps->cr, cfg | CR_AES_ECB | CR_KSE);785 786 /* Set key only after full configuration done */787 stm32_cryp_hw_write_key(cryp);788 789 /* Start prepare key */790 stm32_cryp_enable(cryp);791 /* Wait for end of processing */792 ret = stm32_cryp_wait_busy(cryp);793 if (ret) {794 dev_err(cryp->dev, "Timeout (key preparation)\n");795 return ret;796 }797 798 cfg |= hw_mode | CR_DEC_NOT_ENC;799 800 /* Apply updated config (Decrypt + algo) and flush */801 stm32_cryp_write(cryp, cryp->caps->cr, cfg);802 } else {803 cfg |= hw_mode;804 if (is_decrypt(cryp))805 cfg |= CR_DEC_NOT_ENC;806 807 /* Apply config and flush */808 stm32_cryp_write(cryp, cryp->caps->cr, cfg);809 810 /* Set key only after configuration done */811 stm32_cryp_hw_write_key(cryp);812 }813 814 switch (hw_mode) {815 case CR_AES_GCM:816 case CR_AES_CCM:817 /* Phase 1 : init */818 if (hw_mode == CR_AES_CCM)819 ret = stm32_cryp_ccm_init(cryp, cfg);820 else821 ret = stm32_cryp_gcm_init(cryp, cfg);822 823 if (ret)824 return ret;825 826 break;827 828 case CR_DES_CBC:829 case CR_TDES_CBC:830 case CR_AES_CBC:831 case CR_AES_CTR:832 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv);833 break;834 835 default:836 break;837 }838 839 /* Enable now */840 stm32_cryp_enable(cryp);841 842 return 0;843}844 845static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)846{847 if (!err && (is_gcm(cryp) || is_ccm(cryp)))848 /* Phase 4 : output tag */849 err = stm32_cryp_read_auth_tag(cryp);850 851 if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp))))852 stm32_cryp_get_iv(cryp);853 854 pm_runtime_mark_last_busy(cryp->dev);855 pm_runtime_put_autosuspend(cryp->dev);856 857 if (is_gcm(cryp) || is_ccm(cryp))858 crypto_finalize_aead_request(cryp->engine, cryp->areq, err);859 else860 crypto_finalize_skcipher_request(cryp->engine, cryp->req, err);861}862 863static void stm32_cryp_header_dma_callback(void *param)864{865 struct stm32_cryp *cryp = (struct stm32_cryp *)param;866 int ret;867 u32 reg;868 869 dma_unmap_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE);870 871 reg = stm32_cryp_read(cryp, cryp->caps->dmacr);872 stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN));873 874 kfree(cryp->header_sg);875 876 reg = stm32_cryp_read(cryp, cryp->caps->cr);877 878 if (cryp->header_in) {879 stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN);880 881 ret = stm32_cryp_wait_input(cryp);882 if (ret) {883 dev_err(cryp->dev, "input header ready timeout after dma\n");884 stm32_cryp_finish_req(cryp, ret);885 return;886 }887 stm32_cryp_irq_write_gcmccm_header(cryp);888 WARN_ON(cryp->header_in);889 }890 891 if (stm32_cryp_get_input_text_len(cryp)) {892 /* Phase 3 : payload */893 reg = stm32_cryp_read(cryp, cryp->caps->cr);894 stm32_cryp_write(cryp, cryp->caps->cr, reg & ~CR_CRYPEN);895 896 reg &= ~CR_PH_MASK;897 reg |= CR_PH_PAYLOAD | CR_CRYPEN;898 stm32_cryp_write(cryp, cryp->caps->cr, reg);899 900 if (cryp->flags & FLG_IN_OUT_DMA) {901 ret = stm32_cryp_dma_start(cryp);902 if (ret)903 stm32_cryp_finish_req(cryp, ret);904 } else {905 stm32_cryp_it_start(cryp);906 }907 } else {908 /*909 * Phase 4 : tag.910 * Nothing to read, nothing to write => end request911 */912 stm32_cryp_finish_req(cryp, 0);913 }914}915 916static void stm32_cryp_dma_callback(void *param)917{918 struct stm32_cryp *cryp = (struct stm32_cryp *)param;919 int ret;920 u32 reg;921 922 complete(&cryp->dma_completion); /* completion to indicate no timeout */923 924 dma_sync_sg_for_device(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE);925 926 if (cryp->in_sg != cryp->out_sg)927 dma_unmap_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE);928 929 dma_unmap_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE);930 931 reg = stm32_cryp_read(cryp, cryp->caps->dmacr);932 stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN));933 934 reg = stm32_cryp_read(cryp, cryp->caps->cr);935 936 if (is_gcm(cryp) || is_ccm(cryp)) {937 kfree(cryp->in_sg);938 kfree(cryp->out_sg);939 } else {940 if (cryp->in_sg != cryp->req->src)941 kfree(cryp->in_sg);942 if (cryp->out_sg != cryp->req->dst)943 kfree(cryp->out_sg);944 }945 946 if (cryp->payload_in) {947 stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN);948 949 ret = stm32_cryp_wait_input(cryp);950 if (ret) {951 dev_err(cryp->dev, "input ready timeout after dma\n");952 stm32_cryp_finish_req(cryp, ret);953 return;954 }955 stm32_cryp_irq_write_data(cryp);956 957 ret = stm32_cryp_wait_output(cryp);958 if (ret) {959 dev_err(cryp->dev, "output ready timeout after dma\n");960 stm32_cryp_finish_req(cryp, ret);961 return;962 }963 stm32_cryp_irq_read_data(cryp);964 }965 966 stm32_cryp_finish_req(cryp, 0);967}968 969static int stm32_cryp_header_dma_start(struct stm32_cryp *cryp)970{971 int ret;972 struct dma_async_tx_descriptor *tx_in;973 u32 reg;974 size_t align_size;975 976 ret = dma_map_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE);977 if (!ret) {978 dev_err(cryp->dev, "dma_map_sg() error\n");979 return -ENOMEM;980 }981 982 dma_sync_sg_for_device(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE);983 984 tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->header_sg, cryp->header_sg_len,985 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);986 if (!tx_in) {987 dev_err(cryp->dev, "IN prep_slave_sg() failed\n");988 return -EINVAL;989 }990 991 tx_in->callback_param = cryp;992 tx_in->callback = stm32_cryp_header_dma_callback;993 994 /* Advance scatterwalk to not DMA'ed data */995 align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize);996 scatterwalk_copychunks(NULL, &cryp->in_walk, align_size, 2);997 cryp->header_in -= align_size;998 999 ret = dma_submit_error(dmaengine_submit(tx_in));1000 if (ret < 0) {1001 dev_err(cryp->dev, "DMA in submit failed\n");1002 return ret;1003 }1004 dma_async_issue_pending(cryp->dma_lch_in);1005 1006 reg = stm32_cryp_read(cryp, cryp->caps->dmacr);1007 stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DIEN);1008 1009 return 0;1010}1011 1012static int stm32_cryp_dma_start(struct stm32_cryp *cryp)1013{1014 int ret;1015 size_t align_size;1016 struct dma_async_tx_descriptor *tx_in, *tx_out;1017 u32 reg;1018 1019 if (cryp->in_sg != cryp->out_sg) {1020 ret = dma_map_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE);1021 if (!ret) {1022 dev_err(cryp->dev, "dma_map_sg() error\n");1023 return -ENOMEM;1024 }1025 }1026 1027 ret = dma_map_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE);1028 if (!ret) {1029 dev_err(cryp->dev, "dma_map_sg() error\n");1030 return -ENOMEM;1031 }1032 1033 dma_sync_sg_for_device(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE);1034 1035 tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->in_sg, cryp->in_sg_len,1036 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);1037 if (!tx_in) {1038 dev_err(cryp->dev, "IN prep_slave_sg() failed\n");1039 return -EINVAL;1040 }1041 1042 /* No callback necessary */1043 tx_in->callback_param = cryp;1044 tx_in->callback = NULL;1045 1046 tx_out = dmaengine_prep_slave_sg(cryp->dma_lch_out, cryp->out_sg, cryp->out_sg_len,1047 DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);1048 if (!tx_out) {1049 dev_err(cryp->dev, "OUT prep_slave_sg() failed\n");1050 return -EINVAL;1051 }1052 1053 reinit_completion(&cryp->dma_completion);1054 tx_out->callback = stm32_cryp_dma_callback;1055 tx_out->callback_param = cryp;1056 1057 /* Advance scatterwalk to not DMA'ed data */1058 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize);1059 scatterwalk_copychunks(NULL, &cryp->in_walk, align_size, 2);1060 cryp->payload_in -= align_size;1061 1062 ret = dma_submit_error(dmaengine_submit(tx_in));1063 if (ret < 0) {1064 dev_err(cryp->dev, "DMA in submit failed\n");1065 return ret;1066 }1067 dma_async_issue_pending(cryp->dma_lch_in);1068 1069 /* Advance scatterwalk to not DMA'ed data */1070 scatterwalk_copychunks(NULL, &cryp->out_walk, align_size, 2);1071 cryp->payload_out -= align_size;1072 ret = dma_submit_error(dmaengine_submit(tx_out));1073 if (ret < 0) {1074 dev_err(cryp->dev, "DMA out submit failed\n");1075 return ret;1076 }1077 dma_async_issue_pending(cryp->dma_lch_out);1078 1079 reg = stm32_cryp_read(cryp, cryp->caps->dmacr);1080 stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DOEN | DMACR_DIEN);1081 1082 if (!wait_for_completion_timeout(&cryp->dma_completion, msecs_to_jiffies(1000))) {1083 dev_err(cryp->dev, "DMA out timed out\n");1084 dmaengine_terminate_sync(cryp->dma_lch_out);1085 return -ETIMEDOUT;1086 }1087 1088 return 0;1089}1090 1091static int stm32_cryp_it_start(struct stm32_cryp *cryp)1092{1093 /* Enable interrupt and let the IRQ handler do everything */1094 stm32_cryp_write(cryp, cryp->caps->imsc, IMSCR_IN | IMSCR_OUT);1095 1096 return 0;1097}1098 1099static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq);1100 1101static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm)1102{1103 crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx));1104 1105 return 0;1106}1107 1108static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq);1109 1110static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm)1111{1112 crypto_aead_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx));1113 1114 return 0;1115}1116 1117static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode)1118{1119 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(1120 crypto_skcipher_reqtfm(req));1121 struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req);1122 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);1123 1124 if (!cryp)1125 return -ENODEV;1126 1127 rctx->mode = mode;1128 1129 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);1130}1131 1132static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode)1133{1134 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));1135 struct stm32_cryp_reqctx *rctx = aead_request_ctx(req);1136 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);1137 1138 if (!cryp)1139 return -ENODEV;1140 1141 rctx->mode = mode;1142 1143 return crypto_transfer_aead_request_to_engine(cryp->engine, req);1144}1145 1146static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key,1147 unsigned int keylen)1148{1149 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);1150 1151 memcpy(ctx->key, key, keylen);1152 ctx->keylen = keylen;1153 1154 return 0;1155}1156 1157static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,1158 unsigned int keylen)1159{1160 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&1161 keylen != AES_KEYSIZE_256)1162 return -EINVAL;1163 else1164 return stm32_cryp_setkey(tfm, key, keylen);1165}1166 1167static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key,1168 unsigned int keylen)1169{1170 return verify_skcipher_des_key(tfm, key) ?:1171 stm32_cryp_setkey(tfm, key, keylen);1172}1173 1174static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key,1175 unsigned int keylen)1176{1177 return verify_skcipher_des3_key(tfm, key) ?:1178 stm32_cryp_setkey(tfm, key, keylen);1179}1180 1181static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,1182 unsigned int keylen)1183{1184 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm);1185 1186 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&1187 keylen != AES_KEYSIZE_256)1188 return -EINVAL;1189 1190 memcpy(ctx->key, key, keylen);1191 ctx->keylen = keylen;1192 1193 return 0;1194}1195 1196static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,1197 unsigned int authsize)1198{1199 switch (authsize) {1200 case 4:1201 case 8:1202 case 12:1203 case 13:1204 case 14:1205 case 15:1206 case 16:1207 break;1208 default:1209 return -EINVAL;1210 }1211 1212 return 0;1213}1214 1215static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,1216 unsigned int authsize)1217{1218 switch (authsize) {1219 case 4:1220 case 6:1221 case 8:1222 case 10:1223 case 12:1224 case 14:1225 case 16:1226 break;1227 default:1228 return -EINVAL;1229 }1230 1231 return 0;1232}1233 1234static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req)1235{1236 if (req->cryptlen % AES_BLOCK_SIZE)1237 return -EINVAL;1238 1239 if (req->cryptlen == 0)1240 return 0;1241 1242 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);1243}1244 1245static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req)1246{1247 if (req->cryptlen % AES_BLOCK_SIZE)1248 return -EINVAL;1249 1250 if (req->cryptlen == 0)1251 return 0;1252 1253 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);1254}1255 1256static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req)1257{1258 if (req->cryptlen % AES_BLOCK_SIZE)1259 return -EINVAL;1260 1261 if (req->cryptlen == 0)1262 return 0;1263 1264 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);1265}1266 1267static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req)1268{1269 if (req->cryptlen % AES_BLOCK_SIZE)1270 return -EINVAL;1271 1272 if (req->cryptlen == 0)1273 return 0;1274 1275 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);1276}1277 1278static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req)1279{1280 if (req->cryptlen == 0)1281 return 0;1282 1283 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);1284}1285 1286static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req)1287{1288 if (req->cryptlen == 0)1289 return 0;1290 1291 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);1292}1293 1294static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req)1295{1296 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT);1297}1298 1299static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req)1300{1301 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM);1302}1303 1304static inline int crypto_ccm_check_iv(const u8 *iv)1305{1306 /* 2 <= L <= 8, so 1 <= L' <= 7. */1307 if (iv[0] < 1 || iv[0] > 7)1308 return -EINVAL;1309 1310 return 0;1311}1312 1313static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)1314{1315 int err;1316 1317 err = crypto_ccm_check_iv(req->iv);1318 if (err)1319 return err;1320 1321 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT);1322}1323 1324static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)1325{1326 int err;1327 1328 err = crypto_ccm_check_iv(req->iv);1329 if (err)1330 return err;1331 1332 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM);1333}1334 1335static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req)1336{1337 if (req->cryptlen % DES_BLOCK_SIZE)1338 return -EINVAL;1339 1340 if (req->cryptlen == 0)1341 return 0;1342 1343 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);1344}1345 1346static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req)1347{1348 if (req->cryptlen % DES_BLOCK_SIZE)1349 return -EINVAL;1350 1351 if (req->cryptlen == 0)1352 return 0;1353 1354 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);1355}1356 1357static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req)1358{1359 if (req->cryptlen % DES_BLOCK_SIZE)1360 return -EINVAL;1361 1362 if (req->cryptlen == 0)1363 return 0;1364 1365 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);1366}1367 1368static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req)1369{1370 if (req->cryptlen % DES_BLOCK_SIZE)1371 return -EINVAL;1372 1373 if (req->cryptlen == 0)1374 return 0;1375 1376 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);1377}1378 1379static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req)1380{1381 if (req->cryptlen % DES_BLOCK_SIZE)1382 return -EINVAL;1383 1384 if (req->cryptlen == 0)1385 return 0;1386 1387 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);1388}1389 1390static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req)1391{1392 if (req->cryptlen % DES_BLOCK_SIZE)1393 return -EINVAL;1394 1395 if (req->cryptlen == 0)1396 return 0;1397 1398 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);1399}1400 1401static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req)1402{1403 if (req->cryptlen % DES_BLOCK_SIZE)1404 return -EINVAL;1405 1406 if (req->cryptlen == 0)1407 return 0;1408 1409 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);1410}1411 1412static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req)1413{1414 if (req->cryptlen % DES_BLOCK_SIZE)1415 return -EINVAL;1416 1417 if (req->cryptlen == 0)1418 return 0;1419 1420 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);1421}1422 1423static enum stm32_dma_mode stm32_cryp_dma_check_sg(struct scatterlist *test_sg, size_t len,1424 size_t block_size)1425{1426 struct scatterlist *sg;1427 int i;1428 1429 if (len <= 16)1430 return NO_DMA; /* Faster */1431 1432 for_each_sg(test_sg, sg, sg_nents(test_sg), i) {1433 if (!IS_ALIGNED(sg->length, block_size) && !sg_is_last(sg))1434 return NO_DMA;1435 1436 if (sg->offset % sizeof(u32))1437 return NO_DMA;1438 1439 if (sg_is_last(sg) && !IS_ALIGNED(sg->length, AES_BLOCK_SIZE))1440 return DMA_NEED_SG_TRUNC;1441 }1442 1443 return DMA_PLAIN_SG;1444}1445 1446static enum stm32_dma_mode stm32_cryp_dma_check(struct stm32_cryp *cryp, struct scatterlist *in_sg,1447 struct scatterlist *out_sg)1448{1449 enum stm32_dma_mode ret = DMA_PLAIN_SG;1450 1451 if (!is_aes(cryp))1452 return NO_DMA;1453 1454 if (!cryp->dma_lch_in || !cryp->dma_lch_out)1455 return NO_DMA;1456 1457 ret = stm32_cryp_dma_check_sg(in_sg, cryp->payload_in, AES_BLOCK_SIZE);1458 if (ret == NO_DMA)1459 return ret;1460 1461 ret = stm32_cryp_dma_check_sg(out_sg, cryp->payload_out, AES_BLOCK_SIZE);1462 if (ret == NO_DMA)1463 return ret;1464 1465 /* Check CTR counter overflow */1466 if (is_aes(cryp) && is_ctr(cryp)) {1467 u32 c;1468 __be32 iv3;1469 1470 memcpy(&iv3, &cryp->req->iv[3 * sizeof(u32)], sizeof(iv3));1471 c = be32_to_cpu(iv3);1472 if ((c + cryp->payload_in) < cryp->payload_in)1473 return NO_DMA;1474 }1475 1476 /* Workaround */1477 if (is_aes(cryp) && is_ctr(cryp) && ret == DMA_NEED_SG_TRUNC)1478 return NO_DMA;1479 1480 return ret;1481}1482 1483static int stm32_cryp_truncate_sg(struct scatterlist **new_sg, size_t *new_sg_len,1484 struct scatterlist *sg, off_t skip, size_t size)1485{1486 struct scatterlist *cur;1487 int alloc_sg_len;1488 1489 *new_sg_len = 0;1490 1491 if (!sg || !size) {1492 *new_sg = NULL;1493 return 0;1494 }1495 1496 alloc_sg_len = sg_nents_for_len(sg, skip + size);1497 if (alloc_sg_len < 0)1498 return alloc_sg_len;1499 1500 /* We allocate to much sg entry, but it is easier */1501 *new_sg = kmalloc_array((size_t)alloc_sg_len, sizeof(struct scatterlist), GFP_KERNEL);1502 if (!*new_sg)1503 return -ENOMEM;1504 1505 sg_init_table(*new_sg, (unsigned int)alloc_sg_len);1506 1507 cur = *new_sg;1508 while (sg && size) {1509 unsigned int len = sg->length;1510 unsigned int offset = sg->offset;1511 1512 if (skip > len) {1513 skip -= len;1514 sg = sg_next(sg);1515 continue;1516 }1517 1518 if (skip) {1519 len -= skip;1520 offset += skip;1521 skip = 0;1522 }1523 1524 if (size < len)1525 len = size;1526 1527 if (len > 0) {1528 (*new_sg_len)++;1529 size -= len;1530 sg_set_page(cur, sg_page(sg), len, offset);1531 if (size == 0)1532 sg_mark_end(cur);1533 cur = sg_next(cur);1534 }1535 1536 sg = sg_next(sg);1537 }1538 1539 return 0;1540}1541 1542static int stm32_cryp_cipher_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg,1543 struct scatterlist *out_sg)1544{1545 size_t align_size;1546 int ret;1547 1548 cryp->dma_mode = stm32_cryp_dma_check(cryp, in_sg, out_sg);1549 1550 scatterwalk_start(&cryp->in_walk, in_sg);1551 scatterwalk_start(&cryp->out_walk, out_sg);1552 1553 if (cryp->dma_mode == NO_DMA) {1554 cryp->flags &= ~FLG_IN_OUT_DMA;1555 1556 if (is_ctr(cryp))1557 memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr));1558 1559 } else if (cryp->dma_mode == DMA_NEED_SG_TRUNC) {1560 1561 cryp->flags |= FLG_IN_OUT_DMA;1562 1563 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize);1564 ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, 0, align_size);1565 if (ret)1566 return ret;1567 1568 ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, 0,1569 align_size);1570 if (ret) {1571 kfree(cryp->in_sg);1572 return ret;1573 }1574 } else {1575 cryp->flags |= FLG_IN_OUT_DMA;1576 1577 cryp->in_sg = in_sg;1578 cryp->out_sg = out_sg;1579 1580 ret = sg_nents_for_len(cryp->in_sg, cryp->payload_in);1581 if (ret < 0)1582 return ret;1583 cryp->in_sg_len = (size_t)ret;1584 1585 ret = sg_nents_for_len(out_sg, cryp->payload_out);1586 if (ret < 0)1587 return ret;1588 cryp->out_sg_len = (size_t)ret;1589 }1590 1591 return 0;1592}1593 1594static int stm32_cryp_aead_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg,1595 struct scatterlist *out_sg)1596{1597 size_t align_size;1598 off_t skip;1599 int ret, ret2;1600 1601 cryp->header_sg = NULL;1602 cryp->in_sg = NULL;1603 cryp->out_sg = NULL;1604 1605 if (!cryp->dma_lch_in || !cryp->dma_lch_out) {1606 cryp->dma_mode = NO_DMA;1607 cryp->flags &= ~(FLG_IN_OUT_DMA | FLG_HEADER_DMA);1608 1609 return 0;1610 }1611 1612 /* CCM hw_init may have advanced in header */1613 skip = cryp->areq->assoclen - cryp->header_in;1614 1615 align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize);1616 ret = stm32_cryp_truncate_sg(&cryp->header_sg, &cryp->header_sg_len, in_sg, skip,1617 align_size);1618 if (ret)1619 return ret;1620 1621 ret = stm32_cryp_dma_check_sg(cryp->header_sg, align_size, AES_BLOCK_SIZE);1622 if (ret == NO_DMA) {1623 /* We cannot DMA the header */1624 kfree(cryp->header_sg);1625 cryp->header_sg = NULL;1626 1627 cryp->flags &= ~FLG_HEADER_DMA;1628 } else {1629 cryp->flags |= FLG_HEADER_DMA;1630 }1631 1632 /* Now skip all header to be at payload start */1633 skip = cryp->areq->assoclen;1634 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize);1635 ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, skip, align_size);1636 if (ret) {1637 kfree(cryp->header_sg);1638 return ret;1639 }1640 1641 /* For out buffer align_size is same as in buffer */1642 ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, skip, align_size);1643 if (ret) {1644 kfree(cryp->header_sg);1645 kfree(cryp->in_sg);1646 return ret;1647 }1648 1649 ret = stm32_cryp_dma_check_sg(cryp->in_sg, align_size, AES_BLOCK_SIZE);1650 ret2 = stm32_cryp_dma_check_sg(cryp->out_sg, align_size, AES_BLOCK_SIZE);1651 if (ret == NO_DMA || ret2 == NO_DMA) {1652 kfree(cryp->in_sg);1653 cryp->in_sg = NULL;1654 1655 kfree(cryp->out_sg);1656 cryp->out_sg = NULL;1657 1658 cryp->flags &= ~FLG_IN_OUT_DMA;1659 } else {1660 cryp->flags |= FLG_IN_OUT_DMA;1661 }1662 1663 return 0;1664}1665 1666static int stm32_cryp_prepare_req(struct skcipher_request *req,1667 struct aead_request *areq)1668{1669 struct stm32_cryp_ctx *ctx;1670 struct stm32_cryp *cryp;1671 struct stm32_cryp_reqctx *rctx;1672 struct scatterlist *in_sg, *out_sg;1673 int ret;1674 1675 if (!req && !areq)1676 return -EINVAL;1677 1678 ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :1679 crypto_aead_ctx(crypto_aead_reqtfm(areq));1680 1681 cryp = ctx->cryp;1682 1683 rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);1684 rctx->mode &= FLG_MODE_MASK;1685 1686 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode;1687 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE;1688 cryp->ctx = ctx;1689 1690 if (req) {1691 cryp->req = req;1692 cryp->areq = NULL;1693 cryp->header_in = 0;1694 cryp->payload_in = req->cryptlen;1695 cryp->payload_out = req->cryptlen;1696 cryp->authsize = 0;1697 1698 in_sg = req->src;1699 out_sg = req->dst;1700 1701 ret = stm32_cryp_cipher_prepare(cryp, in_sg, out_sg);1702 if (ret)1703 return ret;1704 1705 ret = stm32_cryp_hw_init(cryp);1706 } else {1707 /*1708 * Length of input and output data:1709 * Encryption case:1710 * INPUT = AssocData || PlainText1711 * <- assoclen -> <- cryptlen ->1712 *1713 * OUTPUT = AssocData || CipherText || AuthTag1714 * <- assoclen -> <-- cryptlen --> <- authsize ->1715 *1716 * Decryption case:1717 * INPUT = AssocData || CipherTex || AuthTag1718 * <- assoclen ---> <---------- cryptlen ---------->1719 *1720 * OUTPUT = AssocData || PlainText1721 * <- assoclen -> <- cryptlen - authsize ->1722 */1723 cryp->areq = areq;1724 cryp->req = NULL;1725 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));1726 if (is_encrypt(cryp)) {1727 cryp->payload_in = areq->cryptlen;1728 cryp->header_in = areq->assoclen;1729 cryp->payload_out = areq->cryptlen;1730 } else {1731 cryp->payload_in = areq->cryptlen - cryp->authsize;1732 cryp->header_in = areq->assoclen;1733 cryp->payload_out = cryp->payload_in;1734 }1735 1736 in_sg = areq->src;1737 out_sg = areq->dst;1738 1739 scatterwalk_start(&cryp->in_walk, in_sg);1740 scatterwalk_start(&cryp->out_walk, out_sg);1741 /* In output, jump after assoc data */1742 scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2);1743 1744 ret = stm32_cryp_hw_init(cryp);1745 if (ret)1746 return ret;1747 1748 ret = stm32_cryp_aead_prepare(cryp, in_sg, out_sg);1749 }1750 1751 return ret;1752}1753 1754static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq)1755{1756 struct skcipher_request *req = container_of(areq,1757 struct skcipher_request,1758 base);1759 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(1760 crypto_skcipher_reqtfm(req));1761 struct stm32_cryp *cryp = ctx->cryp;1762 int ret;1763 1764 if (!cryp)1765 return -ENODEV;1766 1767 ret = stm32_cryp_prepare_req(req, NULL);1768 if (ret)1769 return ret;1770 1771 if (cryp->flags & FLG_IN_OUT_DMA)1772 ret = stm32_cryp_dma_start(cryp);1773 else1774 ret = stm32_cryp_it_start(cryp);1775 1776 if (ret == -ETIMEDOUT)1777 stm32_cryp_finish_req(cryp, ret);1778 1779 return ret;1780}1781 1782static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq)1783{1784 struct aead_request *req = container_of(areq, struct aead_request,1785 base);1786 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));1787 struct stm32_cryp *cryp = ctx->cryp;1788 int err;1789 1790 if (!cryp)1791 return -ENODEV;1792 1793 err = stm32_cryp_prepare_req(NULL, req);1794 if (err)1795 return err;1796 1797 if (!stm32_cryp_get_input_text_len(cryp) && !cryp->header_in &&1798 !(cryp->flags & FLG_HEADER_DMA)) {1799 /* No input data to process: get tag and finish */1800 stm32_cryp_finish_req(cryp, 0);1801 return 0;1802 }1803 1804 if (cryp->flags & FLG_HEADER_DMA)1805 return stm32_cryp_header_dma_start(cryp);1806 1807 if (!cryp->header_in && cryp->flags & FLG_IN_OUT_DMA)1808 return stm32_cryp_dma_start(cryp);1809 1810 return stm32_cryp_it_start(cryp);1811}1812 1813static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)1814{1815 u32 cfg, size_bit;1816 unsigned int i;1817 int ret = 0;1818 1819 /* Update Config */1820 cfg = stm32_cryp_read(cryp, cryp->caps->cr);1821 1822 cfg &= ~CR_PH_MASK;1823 cfg |= CR_PH_FINAL;1824 cfg &= ~CR_DEC_NOT_ENC;1825 cfg |= CR_CRYPEN;1826 1827 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1828 1829 if (is_gcm(cryp)) {1830 /* GCM: write aad and payload size (in bits) */1831 size_bit = cryp->areq->assoclen * 8;1832 if (cryp->caps->swap_final)1833 size_bit = (__force u32)cpu_to_be32(size_bit);1834 1835 stm32_cryp_write(cryp, cryp->caps->din, 0);1836 stm32_cryp_write(cryp, cryp->caps->din, size_bit);1837 1838 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :1839 cryp->areq->cryptlen - cryp->authsize;1840 size_bit *= 8;1841 if (cryp->caps->swap_final)1842 size_bit = (__force u32)cpu_to_be32(size_bit);1843 1844 stm32_cryp_write(cryp, cryp->caps->din, 0);1845 stm32_cryp_write(cryp, cryp->caps->din, size_bit);1846 } else {1847 /* CCM: write CTR0 */1848 u32 iv32[AES_BLOCK_32];1849 u8 *iv = (u8 *)iv32;1850 __be32 *biv = (__be32 *)iv32;1851 1852 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);1853 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);1854 1855 for (i = 0; i < AES_BLOCK_32; i++) {1856 u32 xiv = iv32[i];1857 1858 if (!cryp->caps->padding_wa)1859 xiv = be32_to_cpu(biv[i]);1860 stm32_cryp_write(cryp, cryp->caps->din, xiv);1861 }1862 }1863 1864 /* Wait for output data */1865 ret = stm32_cryp_wait_output(cryp);1866 if (ret) {1867 dev_err(cryp->dev, "Timeout (read tag)\n");1868 return ret;1869 }1870 1871 if (is_encrypt(cryp)) {1872 u32 out_tag[AES_BLOCK_32];1873 1874 /* Get and write tag */1875 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32);1876 scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1);1877 } else {1878 /* Get and check tag */1879 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];1880 1881 scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0);1882 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32);1883 1884 if (crypto_memneq(in_tag, out_tag, cryp->authsize))1885 ret = -EBADMSG;1886 }1887 1888 /* Disable cryp */1889 cfg &= ~CR_CRYPEN;1890 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1891 1892 return ret;1893}1894 1895static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)1896{1897 u32 cr;1898 1899 if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) {1900 /*1901 * In this case, we need to increment manually the ctr counter,1902 * as HW doesn't handle the U32 carry.1903 */1904 crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr));1905 1906 cr = stm32_cryp_read(cryp, cryp->caps->cr);1907 stm32_cryp_write(cryp, cryp->caps->cr, cr & ~CR_CRYPEN);1908 1909 stm32_cryp_hw_write_iv(cryp, cryp->last_ctr);1910 1911 stm32_cryp_write(cryp, cryp->caps->cr, cr);1912 }1913 1914 /* The IV registers are BE */1915 cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l));1916 cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r));1917 cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l));1918 cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r));1919}1920 1921static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp)1922{1923 u32 block[AES_BLOCK_32];1924 1925 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32));1926 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,1927 cryp->payload_out), 1);1928 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,1929 cryp->payload_out);1930}1931 1932static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)1933{1934 u32 block[AES_BLOCK_32] = {0};1935 1936 scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize,1937 cryp->payload_in), 0);1938 writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32));1939 cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in);1940}1941 1942static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)1943{1944 int err;1945 u32 cfg, block[AES_BLOCK_32] = {0};1946 unsigned int i;1947 1948 /* 'Special workaround' procedure described in the datasheet */1949 1950 /* a) disable ip */1951 stm32_cryp_write(cryp, cryp->caps->imsc, 0);1952 cfg = stm32_cryp_read(cryp, cryp->caps->cr);1953 cfg &= ~CR_CRYPEN;1954 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1955 1956 /* b) Update IV1R */1957 stm32_cryp_write(cryp, cryp->caps->iv1r, cryp->gcm_ctr - 2);1958 1959 /* c) change mode to CTR */1960 cfg &= ~CR_ALGO_MASK;1961 cfg |= CR_AES_CTR;1962 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1963 1964 /* a) enable IP */1965 cfg |= CR_CRYPEN;1966 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1967 1968 /* b) pad and write the last block */1969 stm32_cryp_irq_write_block(cryp);1970 /* wait end of process */1971 err = stm32_cryp_wait_output(cryp);1972 if (err) {1973 dev_err(cryp->dev, "Timeout (write gcm last data)\n");1974 return stm32_cryp_finish_req(cryp, err);1975 }1976 1977 /* c) get and store encrypted data */1978 /*1979 * Same code as stm32_cryp_irq_read_data(), but we want to store1980 * block value1981 */1982 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32));1983 1984 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,1985 cryp->payload_out), 1);1986 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,1987 cryp->payload_out);1988 1989 /* d) change mode back to AES GCM */1990 cfg &= ~CR_ALGO_MASK;1991 cfg |= CR_AES_GCM;1992 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1993 1994 /* e) change phase to Final */1995 cfg &= ~CR_PH_MASK;1996 cfg |= CR_PH_FINAL;1997 stm32_cryp_write(cryp, cryp->caps->cr, cfg);1998 1999 /* f) write padded data */2000 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32);2001 2002 /* g) Empty fifo out */2003 err = stm32_cryp_wait_output(cryp);2004 if (err) {2005 dev_err(cryp->dev, "Timeout (write gcm padded data)\n");2006 return stm32_cryp_finish_req(cryp, err);2007 }2008 2009 for (i = 0; i < AES_BLOCK_32; i++)2010 stm32_cryp_read(cryp, cryp->caps->dout);2011 2012 /* h) run the he normal Final phase */2013 stm32_cryp_finish_req(cryp, 0);2014}2015 2016static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)2017{2018 u32 cfg;2019 2020 /* disable ip, set NPBLB and reneable ip */2021 cfg = stm32_cryp_read(cryp, cryp->caps->cr);2022 cfg &= ~CR_CRYPEN;2023 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2024 2025 cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT;2026 cfg |= CR_CRYPEN;2027 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2028}2029 2030static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp)2031{2032 int err = 0;2033 u32 cfg, iv1tmp;2034 u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32];2035 u32 block[AES_BLOCK_32] = {0};2036 unsigned int i;2037 2038 /* 'Special workaround' procedure described in the datasheet */2039 2040 /* a) disable ip */2041 stm32_cryp_write(cryp, cryp->caps->imsc, 0);2042 2043 cfg = stm32_cryp_read(cryp, cryp->caps->cr);2044 cfg &= ~CR_CRYPEN;2045 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2046 2047 /* b) get IV1 from CRYP_CSGCMCCM7 */2048 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4);2049 2050 /* c) Load CRYP_CSGCMCCMxR */2051 for (i = 0; i < ARRAY_SIZE(cstmp1); i++)2052 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);2053 2054 /* d) Write IV1R */2055 stm32_cryp_write(cryp, cryp->caps->iv1r, iv1tmp);2056 2057 /* e) change mode to CTR */2058 cfg &= ~CR_ALGO_MASK;2059 cfg |= CR_AES_CTR;2060 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2061 2062 /* a) enable IP */2063 cfg |= CR_CRYPEN;2064 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2065 2066 /* b) pad and write the last block */2067 stm32_cryp_irq_write_block(cryp);2068 /* wait end of process */2069 err = stm32_cryp_wait_output(cryp);2070 if (err) {2071 dev_err(cryp->dev, "Timeout (write ccm padded data)\n");2072 return stm32_cryp_finish_req(cryp, err);2073 }2074 2075 /* c) get and store decrypted data */2076 /*2077 * Same code as stm32_cryp_irq_read_data(), but we want to store2078 * block value2079 */2080 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32));2081 2082 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,2083 cryp->payload_out), 1);2084 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out);2085 2086 /* d) Load again CRYP_CSGCMCCMxR */2087 for (i = 0; i < ARRAY_SIZE(cstmp2); i++)2088 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4);2089 2090 /* e) change mode back to AES CCM */2091 cfg &= ~CR_ALGO_MASK;2092 cfg |= CR_AES_CCM;2093 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2094 2095 /* f) change phase to header */2096 cfg &= ~CR_PH_MASK;2097 cfg |= CR_PH_HEADER;2098 stm32_cryp_write(cryp, cryp->caps->cr, cfg);2099 2100 /* g) XOR and write padded data */2101 for (i = 0; i < ARRAY_SIZE(block); i++) {2102 block[i] ^= cstmp1[i];2103 block[i] ^= cstmp2[i];2104 stm32_cryp_write(cryp, cryp->caps->din, block[i]);2105 }2106 2107 /* h) wait for completion */2108 err = stm32_cryp_wait_busy(cryp);2109 if (err)2110 dev_err(cryp->dev, "Timeout (write ccm padded data)\n");2111 2112 /* i) run the he normal Final phase */2113 stm32_cryp_finish_req(cryp, err);2114}2115 2116static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)2117{2118 if (unlikely(!cryp->payload_in)) {2119 dev_warn(cryp->dev, "No more data to process\n");2120 return;2121 }2122 2123 if (unlikely(cryp->payload_in < AES_BLOCK_SIZE &&2124 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&2125 is_encrypt(cryp))) {2126 /* Padding for AES GCM encryption */2127 if (cryp->caps->padding_wa) {2128 /* Special case 1 */2129 stm32_cryp_irq_write_gcm_padded_data(cryp);2130 return;2131 }2132 2133 /* Setting padding bytes (NBBLB) */2134 stm32_cryp_irq_set_npblb(cryp);2135 }2136 2137 if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) &&2138 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&2139 is_decrypt(cryp))) {2140 /* Padding for AES CCM decryption */2141 if (cryp->caps->padding_wa) {2142 /* Special case 2 */2143 stm32_cryp_irq_write_ccm_padded_data(cryp);2144 return;2145 }2146 2147 /* Setting padding bytes (NBBLB) */2148 stm32_cryp_irq_set_npblb(cryp);2149 }2150 2151 if (is_aes(cryp) && is_ctr(cryp))2152 stm32_cryp_check_ctr_counter(cryp);2153 2154 stm32_cryp_irq_write_block(cryp);2155}2156 2157static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp)2158{2159 u32 block[AES_BLOCK_32] = {0};2160 size_t written;2161 2162 written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in);2163 2164 scatterwalk_copychunks(block, &cryp->in_walk, written, 0);2165 2166 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32);2167 2168 cryp->header_in -= written;2169 2170 stm32_crypt_gcmccm_end_header(cryp);2171}2172 2173static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)2174{2175 struct stm32_cryp *cryp = arg;2176 u32 ph;2177 u32 it_mask = stm32_cryp_read(cryp, cryp->caps->imsc);2178 2179 if (cryp->irq_status & MISR_OUT)2180 /* Output FIFO IRQ: read data */2181 stm32_cryp_irq_read_data(cryp);2182 2183 if (cryp->irq_status & MISR_IN) {2184 if (is_gcm(cryp) || is_ccm(cryp)) {2185 ph = stm32_cryp_read(cryp, cryp->caps->cr) & CR_PH_MASK;2186 if (unlikely(ph == CR_PH_HEADER))2187 /* Write Header */2188 stm32_cryp_irq_write_gcmccm_header(cryp);2189 else2190 /* Input FIFO IRQ: write data */2191 stm32_cryp_irq_write_data(cryp);2192 if (is_gcm(cryp))2193 cryp->gcm_ctr++;2194 } else {2195 /* Input FIFO IRQ: write data */2196 stm32_cryp_irq_write_data(cryp);2197 }2198 }2199 2200 /* Mask useless interrupts */2201 if (!cryp->payload_in && !cryp->header_in)2202 it_mask &= ~IMSCR_IN;2203 if (!cryp->payload_out)2204 it_mask &= ~IMSCR_OUT;2205 stm32_cryp_write(cryp, cryp->caps->imsc, it_mask);2206 2207 if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) {2208 local_bh_disable();2209 stm32_cryp_finish_req(cryp, 0);2210 local_bh_enable();2211 }2212 2213 return IRQ_HANDLED;2214}2215 2216static irqreturn_t stm32_cryp_irq(int irq, void *arg)2217{2218 struct stm32_cryp *cryp = arg;2219 2220 cryp->irq_status = stm32_cryp_read(cryp, cryp->caps->mis);2221 2222 return IRQ_WAKE_THREAD;2223}2224 2225static int stm32_cryp_dma_init(struct stm32_cryp *cryp)2226{2227 struct dma_slave_config dma_conf;2228 struct dma_chan *chan;2229 int ret;2230 2231 memset(&dma_conf, 0, sizeof(dma_conf));2232 2233 dma_conf.direction = DMA_MEM_TO_DEV;2234 dma_conf.dst_addr = cryp->phys_base + cryp->caps->din;2235 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;2236 dma_conf.dst_maxburst = CRYP_DMA_BURST_REG;2237 dma_conf.device_fc = false;2238 2239 chan = dma_request_chan(cryp->dev, "in");2240 if (IS_ERR(chan))2241 return PTR_ERR(chan);2242 2243 cryp->dma_lch_in = chan;2244 ret = dmaengine_slave_config(cryp->dma_lch_in, &dma_conf);2245 if (ret) {2246 dma_release_channel(cryp->dma_lch_in);2247 cryp->dma_lch_in = NULL;2248 dev_err(cryp->dev, "Couldn't configure DMA in slave.\n");2249 return ret;2250 }2251 2252 memset(&dma_conf, 0, sizeof(dma_conf));2253 2254 dma_conf.direction = DMA_DEV_TO_MEM;2255 dma_conf.src_addr = cryp->phys_base + cryp->caps->dout;2256 dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;2257 dma_conf.src_maxburst = CRYP_DMA_BURST_REG;2258 dma_conf.device_fc = false;2259 2260 chan = dma_request_chan(cryp->dev, "out");2261 if (IS_ERR(chan)) {2262 dma_release_channel(cryp->dma_lch_in);2263 cryp->dma_lch_in = NULL;2264 return PTR_ERR(chan);2265 }2266 2267 cryp->dma_lch_out = chan;2268 2269 ret = dmaengine_slave_config(cryp->dma_lch_out, &dma_conf);2270 if (ret) {2271 dma_release_channel(cryp->dma_lch_out);2272 cryp->dma_lch_out = NULL;2273 dev_err(cryp->dev, "Couldn't configure DMA out slave.\n");2274 dma_release_channel(cryp->dma_lch_in);2275 cryp->dma_lch_in = NULL;2276 return ret;2277 }2278 2279 init_completion(&cryp->dma_completion);2280 2281 return 0;2282}2283 2284static struct skcipher_engine_alg crypto_algs[] = {2285{2286 .base = {2287 .base.cra_name = "ecb(aes)",2288 .base.cra_driver_name = "stm32-ecb-aes",2289 .base.cra_priority = 300,2290 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2291 .base.cra_blocksize = AES_BLOCK_SIZE,2292 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2293 .base.cra_alignmask = 0,2294 .base.cra_module = THIS_MODULE,2295 2296 .init = stm32_cryp_init_tfm,2297 .min_keysize = AES_MIN_KEY_SIZE,2298 .max_keysize = AES_MAX_KEY_SIZE,2299 .setkey = stm32_cryp_aes_setkey,2300 .encrypt = stm32_cryp_aes_ecb_encrypt,2301 .decrypt = stm32_cryp_aes_ecb_decrypt,2302 },2303 .op = {2304 .do_one_request = stm32_cryp_cipher_one_req,2305 },2306},2307{2308 .base = {2309 .base.cra_name = "cbc(aes)",2310 .base.cra_driver_name = "stm32-cbc-aes",2311 .base.cra_priority = 300,2312 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2313 .base.cra_blocksize = AES_BLOCK_SIZE,2314 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2315 .base.cra_alignmask = 0,2316 .base.cra_module = THIS_MODULE,2317 2318 .init = stm32_cryp_init_tfm,2319 .min_keysize = AES_MIN_KEY_SIZE,2320 .max_keysize = AES_MAX_KEY_SIZE,2321 .ivsize = AES_BLOCK_SIZE,2322 .setkey = stm32_cryp_aes_setkey,2323 .encrypt = stm32_cryp_aes_cbc_encrypt,2324 .decrypt = stm32_cryp_aes_cbc_decrypt,2325 },2326 .op = {2327 .do_one_request = stm32_cryp_cipher_one_req,2328 },2329},2330{2331 .base = {2332 .base.cra_name = "ctr(aes)",2333 .base.cra_driver_name = "stm32-ctr-aes",2334 .base.cra_priority = 300,2335 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2336 .base.cra_blocksize = 1,2337 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2338 .base.cra_alignmask = 0,2339 .base.cra_module = THIS_MODULE,2340 2341 .init = stm32_cryp_init_tfm,2342 .min_keysize = AES_MIN_KEY_SIZE,2343 .max_keysize = AES_MAX_KEY_SIZE,2344 .ivsize = AES_BLOCK_SIZE,2345 .setkey = stm32_cryp_aes_setkey,2346 .encrypt = stm32_cryp_aes_ctr_encrypt,2347 .decrypt = stm32_cryp_aes_ctr_decrypt,2348 },2349 .op = {2350 .do_one_request = stm32_cryp_cipher_one_req,2351 },2352},2353{2354 .base = {2355 .base.cra_name = "ecb(des)",2356 .base.cra_driver_name = "stm32-ecb-des",2357 .base.cra_priority = 300,2358 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2359 .base.cra_blocksize = DES_BLOCK_SIZE,2360 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2361 .base.cra_alignmask = 0,2362 .base.cra_module = THIS_MODULE,2363 2364 .init = stm32_cryp_init_tfm,2365 .min_keysize = DES_BLOCK_SIZE,2366 .max_keysize = DES_BLOCK_SIZE,2367 .setkey = stm32_cryp_des_setkey,2368 .encrypt = stm32_cryp_des_ecb_encrypt,2369 .decrypt = stm32_cryp_des_ecb_decrypt,2370 },2371 .op = {2372 .do_one_request = stm32_cryp_cipher_one_req,2373 },2374},2375{2376 .base = {2377 .base.cra_name = "cbc(des)",2378 .base.cra_driver_name = "stm32-cbc-des",2379 .base.cra_priority = 300,2380 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2381 .base.cra_blocksize = DES_BLOCK_SIZE,2382 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2383 .base.cra_alignmask = 0,2384 .base.cra_module = THIS_MODULE,2385 2386 .init = stm32_cryp_init_tfm,2387 .min_keysize = DES_BLOCK_SIZE,2388 .max_keysize = DES_BLOCK_SIZE,2389 .ivsize = DES_BLOCK_SIZE,2390 .setkey = stm32_cryp_des_setkey,2391 .encrypt = stm32_cryp_des_cbc_encrypt,2392 .decrypt = stm32_cryp_des_cbc_decrypt,2393 },2394 .op = {2395 .do_one_request = stm32_cryp_cipher_one_req,2396 },2397},2398{2399 .base = {2400 .base.cra_name = "ecb(des3_ede)",2401 .base.cra_driver_name = "stm32-ecb-des3",2402 .base.cra_priority = 300,2403 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2404 .base.cra_blocksize = DES_BLOCK_SIZE,2405 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2406 .base.cra_alignmask = 0,2407 .base.cra_module = THIS_MODULE,2408 2409 .init = stm32_cryp_init_tfm,2410 .min_keysize = 3 * DES_BLOCK_SIZE,2411 .max_keysize = 3 * DES_BLOCK_SIZE,2412 .setkey = stm32_cryp_tdes_setkey,2413 .encrypt = stm32_cryp_tdes_ecb_encrypt,2414 .decrypt = stm32_cryp_tdes_ecb_decrypt,2415 },2416 .op = {2417 .do_one_request = stm32_cryp_cipher_one_req,2418 },2419},2420{2421 .base = {2422 .base.cra_name = "cbc(des3_ede)",2423 .base.cra_driver_name = "stm32-cbc-des3",2424 .base.cra_priority = 300,2425 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2426 .base.cra_blocksize = DES_BLOCK_SIZE,2427 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),2428 .base.cra_alignmask = 0,2429 .base.cra_module = THIS_MODULE,2430 2431 .init = stm32_cryp_init_tfm,2432 .min_keysize = 3 * DES_BLOCK_SIZE,2433 .max_keysize = 3 * DES_BLOCK_SIZE,2434 .ivsize = DES_BLOCK_SIZE,2435 .setkey = stm32_cryp_tdes_setkey,2436 .encrypt = stm32_cryp_tdes_cbc_encrypt,2437 .decrypt = stm32_cryp_tdes_cbc_decrypt,2438 },2439 .op = {2440 .do_one_request = stm32_cryp_cipher_one_req,2441 },2442},2443};2444 2445static struct aead_engine_alg aead_algs[] = {2446{2447 .base.setkey = stm32_cryp_aes_aead_setkey,2448 .base.setauthsize = stm32_cryp_aes_gcm_setauthsize,2449 .base.encrypt = stm32_cryp_aes_gcm_encrypt,2450 .base.decrypt = stm32_cryp_aes_gcm_decrypt,2451 .base.init = stm32_cryp_aes_aead_init,2452 .base.ivsize = 12,2453 .base.maxauthsize = AES_BLOCK_SIZE,2454 2455 .base.base = {2456 .cra_name = "gcm(aes)",2457 .cra_driver_name = "stm32-gcm-aes",2458 .cra_priority = 300,2459 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2460 .cra_blocksize = 1,2461 .cra_ctxsize = sizeof(struct stm32_cryp_ctx),2462 .cra_alignmask = 0,2463 .cra_module = THIS_MODULE,2464 },2465 .op = {2466 .do_one_request = stm32_cryp_aead_one_req,2467 },2468},2469{2470 .base.setkey = stm32_cryp_aes_aead_setkey,2471 .base.setauthsize = stm32_cryp_aes_ccm_setauthsize,2472 .base.encrypt = stm32_cryp_aes_ccm_encrypt,2473 .base.decrypt = stm32_cryp_aes_ccm_decrypt,2474 .base.init = stm32_cryp_aes_aead_init,2475 .base.ivsize = AES_BLOCK_SIZE,2476 .base.maxauthsize = AES_BLOCK_SIZE,2477 2478 .base.base = {2479 .cra_name = "ccm(aes)",2480 .cra_driver_name = "stm32-ccm-aes",2481 .cra_priority = 300,2482 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY,2483 .cra_blocksize = 1,2484 .cra_ctxsize = sizeof(struct stm32_cryp_ctx),2485 .cra_alignmask = 0,2486 .cra_module = THIS_MODULE,2487 },2488 .op = {2489 .do_one_request = stm32_cryp_aead_one_req,2490 },2491},2492};2493 2494static const struct stm32_cryp_caps ux500_data = {2495 .aeads_support = false,2496 .linear_aes_key = true,2497 .kp_mode = false,2498 .iv_protection = true,2499 .swap_final = true,2500 .padding_wa = true,2501 .cr = UX500_CRYP_CR,2502 .sr = UX500_CRYP_SR,2503 .din = UX500_CRYP_DIN,2504 .dout = UX500_CRYP_DOUT,2505 .dmacr = UX500_CRYP_DMACR,2506 .imsc = UX500_CRYP_IMSC,2507 .mis = UX500_CRYP_MIS,2508 .k1l = UX500_CRYP_K1L,2509 .k1r = UX500_CRYP_K1R,2510 .k3r = UX500_CRYP_K3R,2511 .iv0l = UX500_CRYP_IV0L,2512 .iv0r = UX500_CRYP_IV0R,2513 .iv1l = UX500_CRYP_IV1L,2514 .iv1r = UX500_CRYP_IV1R,2515};2516 2517static const struct stm32_cryp_caps f7_data = {2518 .aeads_support = true,2519 .linear_aes_key = false,2520 .kp_mode = true,2521 .iv_protection = false,2522 .swap_final = true,2523 .padding_wa = true,2524 .cr = CRYP_CR,2525 .sr = CRYP_SR,2526 .din = CRYP_DIN,2527 .dout = CRYP_DOUT,2528 .dmacr = CRYP_DMACR,2529 .imsc = CRYP_IMSCR,2530 .mis = CRYP_MISR,2531 .k1l = CRYP_K1LR,2532 .k1r = CRYP_K1RR,2533 .k3r = CRYP_K3RR,2534 .iv0l = CRYP_IV0LR,2535 .iv0r = CRYP_IV0RR,2536 .iv1l = CRYP_IV1LR,2537 .iv1r = CRYP_IV1RR,2538};2539 2540static const struct stm32_cryp_caps mp1_data = {2541 .aeads_support = true,2542 .linear_aes_key = false,2543 .kp_mode = true,2544 .iv_protection = false,2545 .swap_final = false,2546 .padding_wa = false,2547 .cr = CRYP_CR,2548 .sr = CRYP_SR,2549 .din = CRYP_DIN,2550 .dout = CRYP_DOUT,2551 .dmacr = CRYP_DMACR,2552 .imsc = CRYP_IMSCR,2553 .mis = CRYP_MISR,2554 .k1l = CRYP_K1LR,2555 .k1r = CRYP_K1RR,2556 .k3r = CRYP_K3RR,2557 .iv0l = CRYP_IV0LR,2558 .iv0r = CRYP_IV0RR,2559 .iv1l = CRYP_IV1LR,2560 .iv1r = CRYP_IV1RR,2561};2562 2563static const struct of_device_id stm32_dt_ids[] = {2564 { .compatible = "stericsson,ux500-cryp", .data = &ux500_data},2565 { .compatible = "st,stm32f756-cryp", .data = &f7_data},2566 { .compatible = "st,stm32mp1-cryp", .data = &mp1_data},2567 {},2568};2569MODULE_DEVICE_TABLE(of, stm32_dt_ids);2570 2571static int stm32_cryp_probe(struct platform_device *pdev)2572{2573 struct device *dev = &pdev->dev;2574 struct stm32_cryp *cryp;2575 struct reset_control *rst;2576 int irq, ret;2577 2578 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL);2579 if (!cryp)2580 return -ENOMEM;2581 2582 cryp->caps = of_device_get_match_data(dev);2583 if (!cryp->caps)2584 return -ENODEV;2585 2586 cryp->dev = dev;2587 2588 cryp->regs = devm_platform_ioremap_resource(pdev, 0);2589 if (IS_ERR(cryp->regs))2590 return PTR_ERR(cryp->regs);2591 2592 cryp->phys_base = platform_get_resource(pdev, IORESOURCE_MEM, 0)->start;2593 2594 irq = platform_get_irq(pdev, 0);2595 if (irq < 0)2596 return irq;2597 2598 ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,2599 stm32_cryp_irq_thread, IRQF_ONESHOT,2600 dev_name(dev), cryp);2601 if (ret) {2602 dev_err(dev, "Cannot grab IRQ\n");2603 return ret;2604 }2605 2606 cryp->clk = devm_clk_get(dev, NULL);2607 if (IS_ERR(cryp->clk)) {2608 dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n");2609 2610 return PTR_ERR(cryp->clk);2611 }2612 2613 ret = clk_prepare_enable(cryp->clk);2614 if (ret) {2615 dev_err(cryp->dev, "Failed to enable clock\n");2616 return ret;2617 }2618 2619 pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY);2620 pm_runtime_use_autosuspend(dev);2621 2622 pm_runtime_get_noresume(dev);2623 pm_runtime_set_active(dev);2624 pm_runtime_enable(dev);2625 2626 rst = devm_reset_control_get(dev, NULL);2627 if (IS_ERR(rst)) {2628 ret = PTR_ERR(rst);2629 if (ret == -EPROBE_DEFER)2630 goto err_rst;2631 } else {2632 reset_control_assert(rst);2633 udelay(2);2634 reset_control_deassert(rst);2635 }2636 2637 platform_set_drvdata(pdev, cryp);2638 2639 ret = stm32_cryp_dma_init(cryp);2640 switch (ret) {2641 case 0:2642 break;2643 case -ENODEV:2644 dev_dbg(dev, "DMA mode not available\n");2645 break;2646 default:2647 goto err_dma;2648 }2649 2650 spin_lock(&cryp_list.lock);2651 list_add(&cryp->list, &cryp_list.dev_list);2652 spin_unlock(&cryp_list.lock);2653 2654 /* Initialize crypto engine */2655 cryp->engine = crypto_engine_alloc_init(dev, 1);2656 if (!cryp->engine) {2657 dev_err(dev, "Could not init crypto engine\n");2658 ret = -ENOMEM;2659 goto err_engine1;2660 }2661 2662 ret = crypto_engine_start(cryp->engine);2663 if (ret) {2664 dev_err(dev, "Could not start crypto engine\n");2665 goto err_engine2;2666 }2667 2668 ret = crypto_engine_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));2669 if (ret) {2670 dev_err(dev, "Could not register algs\n");2671 goto err_algs;2672 }2673 2674 if (cryp->caps->aeads_support) {2675 ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs));2676 if (ret)2677 goto err_aead_algs;2678 }2679 2680 dev_info(dev, "Initialized\n");2681 2682 pm_runtime_put_sync(dev);2683 2684 return 0;2685 2686err_aead_algs:2687 crypto_engine_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));2688err_algs:2689err_engine2:2690 crypto_engine_exit(cryp->engine);2691err_engine1:2692 spin_lock(&cryp_list.lock);2693 list_del(&cryp->list);2694 spin_unlock(&cryp_list.lock);2695 2696 if (cryp->dma_lch_in)2697 dma_release_channel(cryp->dma_lch_in);2698 if (cryp->dma_lch_out)2699 dma_release_channel(cryp->dma_lch_out);2700err_dma:2701err_rst:2702 pm_runtime_disable(dev);2703 pm_runtime_put_noidle(dev);2704 2705 clk_disable_unprepare(cryp->clk);2706 2707 return ret;2708}2709 2710static void stm32_cryp_remove(struct platform_device *pdev)2711{2712 struct stm32_cryp *cryp = platform_get_drvdata(pdev);2713 int ret;2714 2715 ret = pm_runtime_get_sync(cryp->dev);2716 2717 if (cryp->caps->aeads_support)2718 crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));2719 crypto_engine_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));2720 2721 crypto_engine_exit(cryp->engine);2722 2723 spin_lock(&cryp_list.lock);2724 list_del(&cryp->list);2725 spin_unlock(&cryp_list.lock);2726 2727 if (cryp->dma_lch_in)2728 dma_release_channel(cryp->dma_lch_in);2729 2730 if (cryp->dma_lch_out)2731 dma_release_channel(cryp->dma_lch_out);2732 2733 pm_runtime_disable(cryp->dev);2734 pm_runtime_put_noidle(cryp->dev);2735 2736 if (ret >= 0)2737 clk_disable_unprepare(cryp->clk);2738}2739 2740#ifdef CONFIG_PM2741static int stm32_cryp_runtime_suspend(struct device *dev)2742{2743 struct stm32_cryp *cryp = dev_get_drvdata(dev);2744 2745 clk_disable_unprepare(cryp->clk);2746 2747 return 0;2748}2749 2750static int stm32_cryp_runtime_resume(struct device *dev)2751{2752 struct stm32_cryp *cryp = dev_get_drvdata(dev);2753 int ret;2754 2755 ret = clk_prepare_enable(cryp->clk);2756 if (ret) {2757 dev_err(cryp->dev, "Failed to prepare_enable clock\n");2758 return ret;2759 }2760 2761 return 0;2762}2763#endif2764 2765static const struct dev_pm_ops stm32_cryp_pm_ops = {2766 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,2767 pm_runtime_force_resume)2768 SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend,2769 stm32_cryp_runtime_resume, NULL)2770};2771 2772static struct platform_driver stm32_cryp_driver = {2773 .probe = stm32_cryp_probe,2774 .remove_new = stm32_cryp_remove,2775 .driver = {2776 .name = DRIVER_NAME,2777 .pm = &stm32_cryp_pm_ops,2778 .of_match_table = stm32_dt_ids,2779 },2780};2781 2782module_platform_driver(stm32_cryp_driver);2783 2784MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");2785MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");2786MODULE_LICENSE("GPL");2787