1008 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Intel Keem Bay OCS ECC Crypto Driver.4 *5 * Copyright (C) 2019-2021 Intel Corporation6 */7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt9 10#include <crypto/ecc_curve.h>11#include <crypto/ecdh.h>12#include <crypto/engine.h>13#include <crypto/internal/ecc.h>14#include <crypto/internal/kpp.h>15#include <crypto/kpp.h>16#include <crypto/rng.h>17#include <linux/clk.h>18#include <linux/completion.h>19#include <linux/err.h>20#include <linux/fips.h>21#include <linux/interrupt.h>22#include <linux/io.h>23#include <linux/iopoll.h>24#include <linux/irq.h>25#include <linux/kernel.h>26#include <linux/module.h>27#include <linux/of.h>28#include <linux/platform_device.h>29#include <linux/scatterlist.h>30#include <linux/string.h>31 32#define DRV_NAME "keembay-ocs-ecc"33 34#define KMB_OCS_ECC_PRIORITY 35035 36#define HW_OFFS_OCS_ECC_COMMAND 0x0000000037#define HW_OFFS_OCS_ECC_STATUS 0x0000000438#define HW_OFFS_OCS_ECC_DATA_IN 0x0000008039#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x0000010040#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x0000018041#define HW_OFFS_OCS_ECC_ISR 0x0000040042#define HW_OFFS_OCS_ECC_IER 0x0000040443 44#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0)45#define HW_OCS_ECC_COMMAND_INS_BP BIT(0)46 47#define HW_OCS_ECC_COMMAND_START_VAL BIT(0)48 49#define OCS_ECC_OP_SIZE_384 BIT(8)50#define OCS_ECC_OP_SIZE_256 051 52/* ECC Instruction : for ECC_COMMAND */53#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP)54#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP)55#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP)56#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP)57#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP)58#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP)59#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP)60#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP)61#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP)62#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP)63 64#define ECC_ENABLE_INTR 165 66#define POLL_USEC 10067#define TIMEOUT_USEC 1000068 69#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS70#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \71 << ECC_DIGITS_TO_BYTES_SHIFT)72 73#define POW_CUBE 374 75/**76 * struct ocs_ecc_dev - ECC device context77 * @list: List of device contexts78 * @dev: OCS ECC device79 * @base_reg: IO base address of OCS ECC80 * @engine: Crypto engine for the device81 * @irq_done: IRQ done completion.82 * @irq: IRQ number83 */84struct ocs_ecc_dev {85 struct list_head list;86 struct device *dev;87 void __iomem *base_reg;88 struct crypto_engine *engine;89 struct completion irq_done;90 int irq;91};92 93/**94 * struct ocs_ecc_ctx - Transformation context.95 * @ecc_dev: The ECC driver associated with this context.96 * @curve: The elliptic curve used by this transformation.97 * @private_key: The private key.98 */99struct ocs_ecc_ctx {100 struct ocs_ecc_dev *ecc_dev;101 const struct ecc_curve *curve;102 u64 private_key[KMB_ECC_VLI_MAX_DIGITS];103};104 105/* Driver data. */106struct ocs_ecc_drv {107 struct list_head dev_list;108 spinlock_t lock; /* Protects dev_list. */109};110 111/* Global variable holding the list of OCS ECC devices (only one expected). */112static struct ocs_ecc_drv ocs_ecc = {113 .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list),114 .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock),115};116 117/* Get OCS ECC tfm context from kpp_request. */118static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req)119{120 return kpp_tfm_ctx(crypto_kpp_reqtfm(req));121}122 123/* Converts number of digits to number of bytes. */124static inline unsigned int digits_to_bytes(unsigned int n)125{126 return n << ECC_DIGITS_TO_BYTES_SHIFT;127}128 129/*130 * Wait for ECC idle i.e when an operation (other than write operations)131 * is done.132 */133static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev)134{135 u32 value;136 137 return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS),138 value,139 !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE),140 POLL_USEC, TIMEOUT_USEC);141}142 143static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size)144{145 iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL,146 ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);147}148 149/* Direct write of u32 buffer to ECC engine with associated instruction. */150static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev,151 u32 op_size,152 u32 inst,153 const void *data_in,154 size_t data_size)155{156 iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);157 158 /* MMIO Write src uint32 to dst. */159 memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in,160 data_size);161}162 163/* Start OCS ECC operation and wait for its completion. */164static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size,165 u32 inst)166{167 reinit_completion(&ecc_dev->irq_done);168 169 iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER);170 iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);171 172 return wait_for_completion_interruptible(&ecc_dev->irq_done);173}174 175/**176 * ocs_ecc_read_cx_out() - Read the CX data output buffer.177 * @dev: The OCS ECC device to read from.178 * @cx_out: The buffer where to store the CX value. Must be at least179 * @byte_count byte long.180 * @byte_count: The amount of data to read.181 */182static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out,183 size_t byte_count)184{185 memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT,186 byte_count);187}188 189/**190 * ocs_ecc_read_cy_out() - Read the CX data output buffer.191 * @dev: The OCS ECC device to read from.192 * @cy_out: The buffer where to store the CY value. Must be at least193 * @byte_count byte long.194 * @byte_count: The amount of data to read.195 */196static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out,197 size_t byte_count)198{199 memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT,200 byte_count);201}202 203static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx)204{205 if (tctx->ecc_dev)206 return tctx->ecc_dev;207 208 spin_lock(&ocs_ecc.lock);209 210 /* Only a single OCS device available. */211 tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev,212 list);213 214 spin_unlock(&ocs_ecc.lock);215 216 return tctx->ecc_dev;217}218 219/* Do point multiplication using OCS ECC HW. */220static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev,221 struct ecc_point *result,222 const struct ecc_point *point,223 u64 *scalar,224 const struct ecc_curve *curve)225{226 u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */227 u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ?228 OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256;229 size_t nbytes = digits_to_bytes(curve->g.ndigits);230 int rc = 0;231 232 /* Generate random nbytes for Simple and Differential SCA protection. */233 rc = crypto_get_default_rng();234 if (rc)235 return rc;236 237 rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes);238 crypto_put_default_rng();239 if (rc)240 return rc;241 242 /* Wait engine to be idle before starting new operation. */243 rc = ocs_ecc_wait_idle(ecc_dev);244 if (rc)245 return rc;246 247 /* Send ecc_start pulse as well as indicating operation size. */248 ocs_ecc_cmd_start(ecc_dev, op_size);249 250 /* Write ax param; Base point (Gx). */251 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX,252 point->x, nbytes);253 254 /* Write ay param; Base point (Gy). */255 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY,256 point->y, nbytes);257 258 /*259 * Write the private key into DATA_IN reg.260 *261 * Since DATA_IN register is used to write different values during the262 * computation private Key value is overwritten with263 * side-channel-resistance value.264 */265 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D,266 scalar, nbytes);267 268 /* Write operand by/l. */269 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L,270 sca, nbytes);271 memzero_explicit(sca, sizeof(sca));272 273 /* Write p = curve prime(GF modulus). */274 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P,275 curve->p, nbytes);276 277 /* Write a = curve coefficient. */278 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A,279 curve->a, nbytes);280 281 /* Make hardware perform the multiplication. */282 rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A);283 if (rc)284 return rc;285 286 /* Read result. */287 ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes);288 ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes);289 290 return 0;291}292 293/**294 * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW.295 * @ecc_dev: The OCS ECC device to use.296 * @scalar_out: Where to store the output scalar.297 * @scalar_a: Input scalar operand 'a'.298 * @scalar_b: Input scalar operand 'b'299 * @curve: The curve on which the operation is performed.300 * @ndigits: The size of the operands (in digits).301 * @inst: The operation to perform (as an OCS ECC instruction).302 *303 * Return: 0 on success, negative error code otherwise.304 */305static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out,306 const u64 *scalar_a, const u64 *scalar_b,307 const struct ecc_curve *curve,308 unsigned int ndigits, const u32 inst)309{310 u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ?311 OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256;312 size_t nbytes = digits_to_bytes(ndigits);313 int rc;314 315 /* Wait engine to be idle before starting new operation. */316 rc = ocs_ecc_wait_idle(ecc_dev);317 if (rc)318 return rc;319 320 /* Send ecc_start pulse as well as indicating operation size. */321 ocs_ecc_cmd_start(ecc_dev, op_size);322 323 /* Write ax param (Base point (Gx).*/324 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX,325 scalar_a, nbytes);326 327 /* Write ay param Base point (Gy).*/328 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY,329 scalar_b, nbytes);330 331 /* Write p = curve prime(GF modulus).*/332 ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P,333 curve->p, nbytes);334 335 /* Give instruction A.B or A+B to ECC engine. */336 rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst);337 if (rc)338 return rc;339 340 ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes);341 342 if (vli_is_zero(scalar_out, ndigits))343 return -EINVAL;344 345 return 0;346}347 348/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */349static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev,350 const struct ecc_curve *curve,351 struct ecc_point *pk)352{353 u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 };354 u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 };355 u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 };356 int rc;357 358 if (WARN_ON(pk->ndigits != curve->g.ndigits))359 return -EINVAL;360 361 /* Check 1: Verify key is not the zero point. */362 if (ecc_point_is_zero(pk))363 return -EINVAL;364 365 /* Check 2: Verify key is in the range [0, p-1]. */366 if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1)367 return -EINVAL;368 369 if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1)370 return -EINVAL;371 372 /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */373 374 /* y^2 */375 /* Compute y^2 -> store in yy */376 rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits,377 OCS_ECC_INST_CALC_A_MUL_B_MODP);378 if (rc)379 goto exit;380 381 /* x^3 */382 /* Assigning w = 3, used for calculating x^3. */383 w[0] = POW_CUBE;384 /* Load the next stage.*/385 rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits,386 OCS_ECC_INST_CALC_A_POW_B_MODP);387 if (rc)388 goto exit;389 390 /* Do a*x -> store in w. */391 rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve,392 pk->ndigits,393 OCS_ECC_INST_CALC_A_MUL_B_MODP);394 if (rc)395 goto exit;396 397 /* Do ax + b == w + b; store in w. */398 rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve,399 pk->ndigits,400 OCS_ECC_INST_CALC_A_ADD_B_MODP);401 if (rc)402 goto exit;403 404 /* x^3 + ax + b == x^3 + w -> store in w. */405 rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits,406 OCS_ECC_INST_CALC_A_ADD_B_MODP);407 if (rc)408 goto exit;409 410 /* Compare y^2 == x^3 + a·x + b. */411 rc = vli_cmp(yy, w, pk->ndigits);412 if (rc)413 rc = -EINVAL;414 415exit:416 memzero_explicit(xxx, sizeof(xxx));417 memzero_explicit(yy, sizeof(yy));418 memzero_explicit(w, sizeof(w));419 420 return rc;421}422 423/* SP800-56A section 5.6.2.3.3 full verification */424static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev,425 const struct ecc_curve *curve,426 struct ecc_point *pk)427{428 struct ecc_point *nQ;429 int rc;430 431 /* Checks 1 through 3 */432 rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk);433 if (rc)434 return rc;435 436 /* Check 4: Verify that nQ is the zero point. */437 nQ = ecc_alloc_point(pk->ndigits);438 if (!nQ)439 return -ENOMEM;440 441 rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve);442 if (rc)443 goto exit;444 445 if (!ecc_point_is_zero(nQ))446 rc = -EINVAL;447 448exit:449 ecc_free_point(nQ);450 451 return rc;452}453 454static int kmb_ecc_is_key_valid(const struct ecc_curve *curve,455 const u64 *private_key, size_t private_key_len)456{457 size_t ndigits = curve->g.ndigits;458 u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1};459 u64 res[KMB_ECC_VLI_MAX_DIGITS];460 461 if (private_key_len != digits_to_bytes(ndigits))462 return -EINVAL;463 464 if (!private_key)465 return -EINVAL;466 467 /* Make sure the private key is in the range [2, n-3]. */468 if (vli_cmp(one, private_key, ndigits) != -1)469 return -EINVAL;470 471 vli_sub(res, curve->n, one, ndigits);472 vli_sub(res, res, one, ndigits);473 if (vli_cmp(res, private_key, ndigits) != 1)474 return -EINVAL;475 476 return 0;477}478 479/*480 * ECC private keys are generated using the method of extra random bits,481 * equivalent to that described in FIPS 186-4, Appendix B.4.1.482 *483 * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer484 * than requested485 * 0 <= c mod(n-1) <= n-2 and implies that486 * 1 <= d <= n-1487 *488 * This method generates a private key uniformly distributed in the range489 * [1, n-1].490 */491static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey)492{493 size_t nbytes = digits_to_bytes(curve->g.ndigits);494 u64 priv[KMB_ECC_VLI_MAX_DIGITS];495 size_t nbits;496 int rc;497 498 nbits = vli_num_bits(curve->n, curve->g.ndigits);499 500 /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */501 if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv))502 return -EINVAL;503 504 /*505 * FIPS 186-4 recommends that the private key should be obtained from a506 * RBG with a security strength equal to or greater than the security507 * strength associated with N.508 *509 * The maximum security strength identified by NIST SP800-57pt1r4 for510 * ECC is 256 (N >= 512).511 *512 * This condition is met by the default RNG because it selects a favored513 * DRBG with a security strength of 256.514 */515 if (crypto_get_default_rng())516 return -EFAULT;517 518 rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes);519 crypto_put_default_rng();520 if (rc)521 goto cleanup;522 523 rc = kmb_ecc_is_key_valid(curve, priv, nbytes);524 if (rc)525 goto cleanup;526 527 ecc_swap_digits(priv, privkey, curve->g.ndigits);528 529cleanup:530 memzero_explicit(&priv, sizeof(priv));531 532 return rc;533}534 535static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,536 unsigned int len)537{538 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);539 struct ecdh params;540 int rc = 0;541 542 rc = crypto_ecdh_decode_key(buf, len, ¶ms);543 if (rc)544 goto cleanup;545 546 /* Ensure key size is not bigger then expected. */547 if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) {548 rc = -EINVAL;549 goto cleanup;550 }551 552 /* Auto-generate private key is not provided. */553 if (!params.key || !params.key_size) {554 rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key);555 goto cleanup;556 }557 558 rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key,559 params.key_size);560 if (rc)561 goto cleanup;562 563 ecc_swap_digits((const u64 *)params.key, tctx->private_key,564 tctx->curve->g.ndigits);565cleanup:566 memzero_explicit(¶ms, sizeof(params));567 568 if (rc)569 tctx->curve = NULL;570 571 return rc;572}573 574/* Compute shared secret. */575static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx,576 struct kpp_request *req)577{578 struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev;579 const struct ecc_curve *curve = tctx->curve;580 u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS];581 u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2];582 size_t copied, nbytes, pubk_len;583 struct ecc_point *pk, *result;584 int rc;585 586 nbytes = digits_to_bytes(curve->g.ndigits);587 588 /* Public key is a point, thus it has two coordinates */589 pubk_len = 2 * nbytes;590 591 /* Copy public key from SG list to pubk_buf. */592 copied = sg_copy_to_buffer(req->src,593 sg_nents_for_len(req->src, pubk_len),594 pubk_buf, pubk_len);595 if (copied != pubk_len)596 return -EINVAL;597 598 /* Allocate and initialize public key point. */599 pk = ecc_alloc_point(curve->g.ndigits);600 if (!pk)601 return -ENOMEM;602 603 ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits);604 ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits);605 606 /*607 * Check the public key for following608 * Check 1: Verify key is not the zero point.609 * Check 2: Verify key is in the range [1, p-1].610 * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p611 */612 rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk);613 if (rc)614 goto exit_free_pk;615 616 /* Allocate point for storing computed shared secret. */617 result = ecc_alloc_point(pk->ndigits);618 if (!result) {619 rc = -ENOMEM;620 goto exit_free_pk;621 }622 623 /* Calculate the shared secret.*/624 rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve);625 if (rc)626 goto exit_free_result;627 628 if (ecc_point_is_zero(result)) {629 rc = -EFAULT;630 goto exit_free_result;631 }632 633 /* Copy shared secret from point to buffer. */634 ecc_swap_digits(result->x, shared_secret, result->ndigits);635 636 /* Request might ask for less bytes than what we have. */637 nbytes = min_t(size_t, nbytes, req->dst_len);638 639 copied = sg_copy_from_buffer(req->dst,640 sg_nents_for_len(req->dst, nbytes),641 shared_secret, nbytes);642 643 if (copied != nbytes)644 rc = -EINVAL;645 646 memzero_explicit(shared_secret, sizeof(shared_secret));647 648exit_free_result:649 ecc_free_point(result);650 651exit_free_pk:652 ecc_free_point(pk);653 654 return rc;655}656 657/* Compute public key. */658static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx,659 struct kpp_request *req)660{661 const struct ecc_curve *curve = tctx->curve;662 u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2];663 struct ecc_point *pk;664 size_t pubk_len;665 size_t copied;666 int rc;667 668 /* Public key is a point, so it has double the digits. */669 pubk_len = 2 * digits_to_bytes(curve->g.ndigits);670 671 pk = ecc_alloc_point(curve->g.ndigits);672 if (!pk)673 return -ENOMEM;674 675 /* Public Key(pk) = priv * G. */676 rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key,677 curve);678 if (rc)679 goto exit;680 681 /* SP800-56A rev 3 5.6.2.1.3 key check */682 if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) {683 rc = -EAGAIN;684 goto exit;685 }686 687 /* Copy public key from point to buffer. */688 ecc_swap_digits(pk->x, pubk_buf, pk->ndigits);689 ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits);690 691 /* Copy public key to req->dst. */692 copied = sg_copy_from_buffer(req->dst,693 sg_nents_for_len(req->dst, pubk_len),694 pubk_buf, pubk_len);695 696 if (copied != pubk_len)697 rc = -EINVAL;698 699exit:700 ecc_free_point(pk);701 702 return rc;703}704 705static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine,706 void *areq)707{708 struct kpp_request *req = container_of(areq, struct kpp_request, base);709 struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);710 struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev;711 int rc;712 713 if (req->src)714 rc = kmb_ecc_do_shared_secret(tctx, req);715 else716 rc = kmb_ecc_do_public_key(tctx, req);717 718 crypto_finalize_kpp_request(ecc_dev->engine, req, rc);719 720 return 0;721}722 723static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req)724{725 struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);726 const struct ecc_curve *curve = tctx->curve;727 728 /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */729 if (!tctx->curve)730 return -EINVAL;731 732 /* Ensure dst is present. */733 if (!req->dst)734 return -EINVAL;735 736 /* Check the request dst is big enough to hold the public key. */737 if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits)))738 return -EINVAL;739 740 /* 'src' is not supposed to be present when generate pubk is called. */741 if (req->src)742 return -EINVAL;743 744 return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine,745 req);746}747 748static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req)749{750 struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);751 const struct ecc_curve *curve = tctx->curve;752 753 /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */754 if (!tctx->curve)755 return -EINVAL;756 757 /* Ensure dst is present. */758 if (!req->dst)759 return -EINVAL;760 761 /* Ensure src is present. */762 if (!req->src)763 return -EINVAL;764 765 /*766 * req->src is expected to the (other-side) public key, so its length767 * must be 2 * coordinate size (in bytes).768 */769 if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits))770 return -EINVAL;771 772 return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine,773 req);774}775 776static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id)777{778 memset(tctx, 0, sizeof(*tctx));779 780 tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx);781 782 if (IS_ERR(tctx->ecc_dev)) {783 pr_err("Failed to find the device : %ld\n",784 PTR_ERR(tctx->ecc_dev));785 return PTR_ERR(tctx->ecc_dev);786 }787 788 tctx->curve = ecc_get_curve(curve_id);789 if (!tctx->curve)790 return -EOPNOTSUPP;791 792 return 0;793}794 795static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)796{797 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);798 799 return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256);800}801 802static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)803{804 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);805 806 return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384);807}808 809static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm)810{811 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);812 813 memzero_explicit(tctx->private_key, sizeof(*tctx->private_key));814}815 816static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm)817{818 struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);819 820 /* Public key is made of two coordinates, so double the digits. */821 return digits_to_bytes(tctx->curve->g.ndigits) * 2;822}823 824static struct kpp_engine_alg ocs_ecdh_p256 = {825 .base.set_secret = kmb_ocs_ecdh_set_secret,826 .base.generate_public_key = kmb_ocs_ecdh_generate_public_key,827 .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret,828 .base.init = kmb_ocs_ecdh_nist_p256_init_tfm,829 .base.exit = kmb_ocs_ecdh_exit_tfm,830 .base.max_size = kmb_ocs_ecdh_max_size,831 .base.base = {832 .cra_name = "ecdh-nist-p256",833 .cra_driver_name = "ecdh-nist-p256-keembay-ocs",834 .cra_priority = KMB_OCS_ECC_PRIORITY,835 .cra_module = THIS_MODULE,836 .cra_ctxsize = sizeof(struct ocs_ecc_ctx),837 },838 .op.do_one_request = kmb_ocs_ecc_do_one_request,839};840 841static struct kpp_engine_alg ocs_ecdh_p384 = {842 .base.set_secret = kmb_ocs_ecdh_set_secret,843 .base.generate_public_key = kmb_ocs_ecdh_generate_public_key,844 .base.compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret,845 .base.init = kmb_ocs_ecdh_nist_p384_init_tfm,846 .base.exit = kmb_ocs_ecdh_exit_tfm,847 .base.max_size = kmb_ocs_ecdh_max_size,848 .base.base = {849 .cra_name = "ecdh-nist-p384",850 .cra_driver_name = "ecdh-nist-p384-keembay-ocs",851 .cra_priority = KMB_OCS_ECC_PRIORITY,852 .cra_module = THIS_MODULE,853 .cra_ctxsize = sizeof(struct ocs_ecc_ctx),854 },855 .op.do_one_request = kmb_ocs_ecc_do_one_request,856};857 858static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id)859{860 struct ocs_ecc_dev *ecc_dev = dev_id;861 u32 status;862 863 /*864 * Read the status register and write it back to clear the865 * DONE_INT_STATUS bit.866 */867 status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR);868 iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR);869 870 if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE))871 return IRQ_NONE;872 873 complete(&ecc_dev->irq_done);874 875 return IRQ_HANDLED;876}877 878static int kmb_ocs_ecc_probe(struct platform_device *pdev)879{880 struct device *dev = &pdev->dev;881 struct ocs_ecc_dev *ecc_dev;882 int rc;883 884 ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL);885 if (!ecc_dev)886 return -ENOMEM;887 888 ecc_dev->dev = dev;889 890 platform_set_drvdata(pdev, ecc_dev);891 892 INIT_LIST_HEAD(&ecc_dev->list);893 init_completion(&ecc_dev->irq_done);894 895 /* Get base register address. */896 ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);897 if (IS_ERR(ecc_dev->base_reg)) {898 dev_err(dev, "Failed to get base address\n");899 rc = PTR_ERR(ecc_dev->base_reg);900 goto list_del;901 }902 903 /* Get and request IRQ */904 ecc_dev->irq = platform_get_irq(pdev, 0);905 if (ecc_dev->irq < 0) {906 rc = ecc_dev->irq;907 goto list_del;908 }909 910 rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler,911 NULL, 0, "keembay-ocs-ecc", ecc_dev);912 if (rc < 0) {913 dev_err(dev, "Could not request IRQ\n");914 goto list_del;915 }916 917 /* Add device to the list of OCS ECC devices. */918 spin_lock(&ocs_ecc.lock);919 list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list);920 spin_unlock(&ocs_ecc.lock);921 922 /* Initialize crypto engine. */923 ecc_dev->engine = crypto_engine_alloc_init(dev, 1);924 if (!ecc_dev->engine) {925 dev_err(dev, "Could not allocate crypto engine\n");926 rc = -ENOMEM;927 goto list_del;928 }929 930 rc = crypto_engine_start(ecc_dev->engine);931 if (rc) {932 dev_err(dev, "Could not start crypto engine\n");933 goto cleanup;934 }935 936 /* Register the KPP algo. */937 rc = crypto_engine_register_kpp(&ocs_ecdh_p256);938 if (rc) {939 dev_err(dev,940 "Could not register OCS algorithms with Crypto API\n");941 goto cleanup;942 }943 944 rc = crypto_engine_register_kpp(&ocs_ecdh_p384);945 if (rc) {946 dev_err(dev,947 "Could not register OCS algorithms with Crypto API\n");948 goto ocs_ecdh_p384_error;949 }950 951 return 0;952 953ocs_ecdh_p384_error:954 crypto_engine_unregister_kpp(&ocs_ecdh_p256);955 956cleanup:957 crypto_engine_exit(ecc_dev->engine);958 959list_del:960 spin_lock(&ocs_ecc.lock);961 list_del(&ecc_dev->list);962 spin_unlock(&ocs_ecc.lock);963 964 return rc;965}966 967static void kmb_ocs_ecc_remove(struct platform_device *pdev)968{969 struct ocs_ecc_dev *ecc_dev;970 971 ecc_dev = platform_get_drvdata(pdev);972 973 crypto_engine_unregister_kpp(&ocs_ecdh_p384);974 crypto_engine_unregister_kpp(&ocs_ecdh_p256);975 976 spin_lock(&ocs_ecc.lock);977 list_del(&ecc_dev->list);978 spin_unlock(&ocs_ecc.lock);979 980 crypto_engine_exit(ecc_dev->engine);981}982 983/* Device tree driver match. */984static const struct of_device_id kmb_ocs_ecc_of_match[] = {985 {986 .compatible = "intel,keembay-ocs-ecc",987 },988 {}989};990 991/* The OCS driver is a platform device. */992static struct platform_driver kmb_ocs_ecc_driver = {993 .probe = kmb_ocs_ecc_probe,994 .remove_new = kmb_ocs_ecc_remove,995 .driver = {996 .name = DRV_NAME,997 .of_match_table = kmb_ocs_ecc_of_match,998 },999};1000module_platform_driver(kmb_ocs_ecc_driver);1001 1002MODULE_LICENSE("GPL");1003MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver");1004MODULE_ALIAS_CRYPTO("ecdh-nist-p256");1005MODULE_ALIAS_CRYPTO("ecdh-nist-p384");1006MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs");1007MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs");1008