742 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Handle async block request by crypto hardware engine.4 *5 * Copyright (C) 2016 Linaro, Inc.6 *7 * Author: Baolin Wang <baolin.wang@linaro.org>8 */9 10#include <crypto/internal/aead.h>11#include <crypto/internal/akcipher.h>12#include <crypto/internal/engine.h>13#include <crypto/internal/hash.h>14#include <crypto/internal/kpp.h>15#include <crypto/internal/skcipher.h>16#include <linux/err.h>17#include <linux/delay.h>18#include <linux/device.h>19#include <linux/kernel.h>20#include <linux/module.h>21#include <uapi/linux/sched/types.h>22#include "internal.h"23 24#define CRYPTO_ENGINE_MAX_QLEN 1025 26/* Temporary algorithm flag used to indicate an updated driver. */27#define CRYPTO_ALG_ENGINE 0x20028 29struct crypto_engine_alg {30 struct crypto_alg base;31 struct crypto_engine_op op;32};33 34/**35 * crypto_finalize_request - finalize one request if the request is done36 * @engine: the hardware engine37 * @req: the request need to be finalized38 * @err: error number39 */40static void crypto_finalize_request(struct crypto_engine *engine,41 struct crypto_async_request *req, int err)42{43 unsigned long flags;44 45 /*46 * If hardware cannot enqueue more requests47 * and retry mechanism is not supported48 * make sure we are completing the current request49 */50 if (!engine->retry_support) {51 spin_lock_irqsave(&engine->queue_lock, flags);52 if (engine->cur_req == req) {53 engine->cur_req = NULL;54 }55 spin_unlock_irqrestore(&engine->queue_lock, flags);56 }57 58 lockdep_assert_in_softirq();59 crypto_request_complete(req, err);60 61 kthread_queue_work(engine->kworker, &engine->pump_requests);62}63 64/**65 * crypto_pump_requests - dequeue one request from engine queue to process66 * @engine: the hardware engine67 * @in_kthread: true if we are in the context of the request pump thread68 *69 * This function checks if there is any request in the engine queue that70 * needs processing and if so call out to the driver to initialize hardware71 * and handle each request.72 */73static void crypto_pump_requests(struct crypto_engine *engine,74 bool in_kthread)75{76 struct crypto_async_request *async_req, *backlog;77 struct crypto_engine_alg *alg;78 struct crypto_engine_op *op;79 unsigned long flags;80 bool was_busy = false;81 int ret;82 83 spin_lock_irqsave(&engine->queue_lock, flags);84 85 /* Make sure we are not already running a request */86 if (!engine->retry_support && engine->cur_req)87 goto out;88 89 /* If another context is idling then defer */90 if (engine->idling) {91 kthread_queue_work(engine->kworker, &engine->pump_requests);92 goto out;93 }94 95 /* Check if the engine queue is idle */96 if (!crypto_queue_len(&engine->queue) || !engine->running) {97 if (!engine->busy)98 goto out;99 100 /* Only do teardown in the thread */101 if (!in_kthread) {102 kthread_queue_work(engine->kworker,103 &engine->pump_requests);104 goto out;105 }106 107 engine->busy = false;108 engine->idling = true;109 spin_unlock_irqrestore(&engine->queue_lock, flags);110 111 if (engine->unprepare_crypt_hardware &&112 engine->unprepare_crypt_hardware(engine))113 dev_err(engine->dev, "failed to unprepare crypt hardware\n");114 115 spin_lock_irqsave(&engine->queue_lock, flags);116 engine->idling = false;117 goto out;118 }119 120start_request:121 /* Get the fist request from the engine queue to handle */122 backlog = crypto_get_backlog(&engine->queue);123 async_req = crypto_dequeue_request(&engine->queue);124 if (!async_req)125 goto out;126 127 /*128 * If hardware doesn't support the retry mechanism,129 * keep track of the request we are processing now.130 * We'll need it on completion (crypto_finalize_request).131 */132 if (!engine->retry_support)133 engine->cur_req = async_req;134 135 if (engine->busy)136 was_busy = true;137 else138 engine->busy = true;139 140 spin_unlock_irqrestore(&engine->queue_lock, flags);141 142 /* Until here we get the request need to be encrypted successfully */143 if (!was_busy && engine->prepare_crypt_hardware) {144 ret = engine->prepare_crypt_hardware(engine);145 if (ret) {146 dev_err(engine->dev, "failed to prepare crypt hardware\n");147 goto req_err_1;148 }149 }150 151 if (async_req->tfm->__crt_alg->cra_flags & CRYPTO_ALG_ENGINE) {152 alg = container_of(async_req->tfm->__crt_alg,153 struct crypto_engine_alg, base);154 op = &alg->op;155 } else {156 dev_err(engine->dev, "failed to do request\n");157 ret = -EINVAL;158 goto req_err_1;159 }160 161 ret = op->do_one_request(engine, async_req);162 163 /* Request unsuccessfully executed by hardware */164 if (ret < 0) {165 /*166 * If hardware queue is full (-ENOSPC), requeue request167 * regardless of backlog flag.168 * Otherwise, unprepare and complete the request.169 */170 if (!engine->retry_support ||171 (ret != -ENOSPC)) {172 dev_err(engine->dev,173 "Failed to do one request from queue: %d\n",174 ret);175 goto req_err_1;176 }177 spin_lock_irqsave(&engine->queue_lock, flags);178 /*179 * If hardware was unable to execute request, enqueue it180 * back in front of crypto-engine queue, to keep the order181 * of requests.182 */183 crypto_enqueue_request_head(&engine->queue, async_req);184 185 kthread_queue_work(engine->kworker, &engine->pump_requests);186 goto out;187 }188 189 goto retry;190 191req_err_1:192 crypto_request_complete(async_req, ret);193 194retry:195 if (backlog)196 crypto_request_complete(backlog, -EINPROGRESS);197 198 /* If retry mechanism is supported, send new requests to engine */199 if (engine->retry_support) {200 spin_lock_irqsave(&engine->queue_lock, flags);201 goto start_request;202 }203 return;204 205out:206 spin_unlock_irqrestore(&engine->queue_lock, flags);207 208 /*209 * Batch requests is possible only if210 * hardware can enqueue multiple requests211 */212 if (engine->do_batch_requests) {213 ret = engine->do_batch_requests(engine);214 if (ret)215 dev_err(engine->dev, "failed to do batch requests: %d\n",216 ret);217 }218 219 return;220}221 222static void crypto_pump_work(struct kthread_work *work)223{224 struct crypto_engine *engine =225 container_of(work, struct crypto_engine, pump_requests);226 227 crypto_pump_requests(engine, true);228}229 230/**231 * crypto_transfer_request - transfer the new request into the engine queue232 * @engine: the hardware engine233 * @req: the request need to be listed into the engine queue234 * @need_pump: indicates whether queue the pump of request to kthread_work235 */236static int crypto_transfer_request(struct crypto_engine *engine,237 struct crypto_async_request *req,238 bool need_pump)239{240 unsigned long flags;241 int ret;242 243 spin_lock_irqsave(&engine->queue_lock, flags);244 245 if (!engine->running) {246 spin_unlock_irqrestore(&engine->queue_lock, flags);247 return -ESHUTDOWN;248 }249 250 ret = crypto_enqueue_request(&engine->queue, req);251 252 if (!engine->busy && need_pump)253 kthread_queue_work(engine->kworker, &engine->pump_requests);254 255 spin_unlock_irqrestore(&engine->queue_lock, flags);256 return ret;257}258 259/**260 * crypto_transfer_request_to_engine - transfer one request to list261 * into the engine queue262 * @engine: the hardware engine263 * @req: the request need to be listed into the engine queue264 */265static int crypto_transfer_request_to_engine(struct crypto_engine *engine,266 struct crypto_async_request *req)267{268 return crypto_transfer_request(engine, req, true);269}270 271/**272 * crypto_transfer_aead_request_to_engine - transfer one aead_request273 * to list into the engine queue274 * @engine: the hardware engine275 * @req: the request need to be listed into the engine queue276 */277int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,278 struct aead_request *req)279{280 return crypto_transfer_request_to_engine(engine, &req->base);281}282EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);283 284/**285 * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request286 * to list into the engine queue287 * @engine: the hardware engine288 * @req: the request need to be listed into the engine queue289 */290int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,291 struct akcipher_request *req)292{293 return crypto_transfer_request_to_engine(engine, &req->base);294}295EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);296 297/**298 * crypto_transfer_hash_request_to_engine - transfer one ahash_request299 * to list into the engine queue300 * @engine: the hardware engine301 * @req: the request need to be listed into the engine queue302 */303int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,304 struct ahash_request *req)305{306 return crypto_transfer_request_to_engine(engine, &req->base);307}308EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);309 310/**311 * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list312 * into the engine queue313 * @engine: the hardware engine314 * @req: the request need to be listed into the engine queue315 */316int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine,317 struct kpp_request *req)318{319 return crypto_transfer_request_to_engine(engine, &req->base);320}321EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine);322 323/**324 * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request325 * to list into the engine queue326 * @engine: the hardware engine327 * @req: the request need to be listed into the engine queue328 */329int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,330 struct skcipher_request *req)331{332 return crypto_transfer_request_to_engine(engine, &req->base);333}334EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);335 336/**337 * crypto_finalize_aead_request - finalize one aead_request if338 * the request is done339 * @engine: the hardware engine340 * @req: the request need to be finalized341 * @err: error number342 */343void crypto_finalize_aead_request(struct crypto_engine *engine,344 struct aead_request *req, int err)345{346 return crypto_finalize_request(engine, &req->base, err);347}348EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);349 350/**351 * crypto_finalize_akcipher_request - finalize one akcipher_request if352 * the request is done353 * @engine: the hardware engine354 * @req: the request need to be finalized355 * @err: error number356 */357void crypto_finalize_akcipher_request(struct crypto_engine *engine,358 struct akcipher_request *req, int err)359{360 return crypto_finalize_request(engine, &req->base, err);361}362EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);363 364/**365 * crypto_finalize_hash_request - finalize one ahash_request if366 * the request is done367 * @engine: the hardware engine368 * @req: the request need to be finalized369 * @err: error number370 */371void crypto_finalize_hash_request(struct crypto_engine *engine,372 struct ahash_request *req, int err)373{374 return crypto_finalize_request(engine, &req->base, err);375}376EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);377 378/**379 * crypto_finalize_kpp_request - finalize one kpp_request if the request is done380 * @engine: the hardware engine381 * @req: the request need to be finalized382 * @err: error number383 */384void crypto_finalize_kpp_request(struct crypto_engine *engine,385 struct kpp_request *req, int err)386{387 return crypto_finalize_request(engine, &req->base, err);388}389EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request);390 391/**392 * crypto_finalize_skcipher_request - finalize one skcipher_request if393 * the request is done394 * @engine: the hardware engine395 * @req: the request need to be finalized396 * @err: error number397 */398void crypto_finalize_skcipher_request(struct crypto_engine *engine,399 struct skcipher_request *req, int err)400{401 return crypto_finalize_request(engine, &req->base, err);402}403EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);404 405/**406 * crypto_engine_start - start the hardware engine407 * @engine: the hardware engine need to be started408 *409 * Return 0 on success, else on fail.410 */411int crypto_engine_start(struct crypto_engine *engine)412{413 unsigned long flags;414 415 spin_lock_irqsave(&engine->queue_lock, flags);416 417 if (engine->running || engine->busy) {418 spin_unlock_irqrestore(&engine->queue_lock, flags);419 return -EBUSY;420 }421 422 engine->running = true;423 spin_unlock_irqrestore(&engine->queue_lock, flags);424 425 kthread_queue_work(engine->kworker, &engine->pump_requests);426 427 return 0;428}429EXPORT_SYMBOL_GPL(crypto_engine_start);430 431/**432 * crypto_engine_stop - stop the hardware engine433 * @engine: the hardware engine need to be stopped434 *435 * Return 0 on success, else on fail.436 */437int crypto_engine_stop(struct crypto_engine *engine)438{439 unsigned long flags;440 unsigned int limit = 500;441 int ret = 0;442 443 spin_lock_irqsave(&engine->queue_lock, flags);444 445 /*446 * If the engine queue is not empty or the engine is on busy state,447 * we need to wait for a while to pump the requests of engine queue.448 */449 while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {450 spin_unlock_irqrestore(&engine->queue_lock, flags);451 msleep(20);452 spin_lock_irqsave(&engine->queue_lock, flags);453 }454 455 if (crypto_queue_len(&engine->queue) || engine->busy)456 ret = -EBUSY;457 else458 engine->running = false;459 460 spin_unlock_irqrestore(&engine->queue_lock, flags);461 462 if (ret)463 dev_warn(engine->dev, "could not stop engine\n");464 465 return ret;466}467EXPORT_SYMBOL_GPL(crypto_engine_stop);468 469/**470 * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure471 * and initialize it by setting the maximum number of entries in the software472 * crypto-engine queue.473 * @dev: the device attached with one hardware engine474 * @retry_support: whether hardware has support for retry mechanism475 * @cbk_do_batch: pointer to a callback function to be invoked when executing476 * a batch of requests.477 * This has the form:478 * callback(struct crypto_engine *engine)479 * where:480 * engine: the crypto engine structure.481 * @rt: whether this queue is set to run as a realtime task482 * @qlen: maximum size of the crypto-engine queue483 *484 * This must be called from context that can sleep.485 * Return: the crypto engine structure on success, else NULL.486 */487struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,488 bool retry_support,489 int (*cbk_do_batch)(struct crypto_engine *engine),490 bool rt, int qlen)491{492 struct crypto_engine *engine;493 494 if (!dev)495 return NULL;496 497 engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);498 if (!engine)499 return NULL;500 501 engine->dev = dev;502 engine->rt = rt;503 engine->running = false;504 engine->busy = false;505 engine->idling = false;506 engine->retry_support = retry_support;507 engine->priv_data = dev;508 /*509 * Batch requests is possible only if510 * hardware has support for retry mechanism.511 */512 engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;513 514 snprintf(engine->name, sizeof(engine->name),515 "%s-engine", dev_name(dev));516 517 crypto_init_queue(&engine->queue, qlen);518 spin_lock_init(&engine->queue_lock);519 520 engine->kworker = kthread_create_worker(0, "%s", engine->name);521 if (IS_ERR(engine->kworker)) {522 dev_err(dev, "failed to create crypto request pump task\n");523 return NULL;524 }525 kthread_init_work(&engine->pump_requests, crypto_pump_work);526 527 if (engine->rt) {528 dev_info(dev, "will run requests pump with realtime priority\n");529 sched_set_fifo(engine->kworker->task);530 }531 532 return engine;533}534EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);535 536/**537 * crypto_engine_alloc_init - allocate crypto hardware engine structure and538 * initialize it.539 * @dev: the device attached with one hardware engine540 * @rt: whether this queue is set to run as a realtime task541 *542 * This must be called from context that can sleep.543 * Return: the crypto engine structure on success, else NULL.544 */545struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)546{547 return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,548 CRYPTO_ENGINE_MAX_QLEN);549}550EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);551 552/**553 * crypto_engine_exit - free the resources of hardware engine when exit554 * @engine: the hardware engine need to be freed555 */556void crypto_engine_exit(struct crypto_engine *engine)557{558 int ret;559 560 ret = crypto_engine_stop(engine);561 if (ret)562 return;563 564 kthread_destroy_worker(engine->kworker);565}566EXPORT_SYMBOL_GPL(crypto_engine_exit);567 568int crypto_engine_register_aead(struct aead_engine_alg *alg)569{570 if (!alg->op.do_one_request)571 return -EINVAL;572 573 alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;574 575 return crypto_register_aead(&alg->base);576}577EXPORT_SYMBOL_GPL(crypto_engine_register_aead);578 579void crypto_engine_unregister_aead(struct aead_engine_alg *alg)580{581 crypto_unregister_aead(&alg->base);582}583EXPORT_SYMBOL_GPL(crypto_engine_unregister_aead);584 585int crypto_engine_register_aeads(struct aead_engine_alg *algs, int count)586{587 int i, ret;588 589 for (i = 0; i < count; i++) {590 ret = crypto_engine_register_aead(&algs[i]);591 if (ret)592 goto err;593 }594 595 return 0;596 597err:598 crypto_engine_unregister_aeads(algs, i);599 600 return ret;601}602EXPORT_SYMBOL_GPL(crypto_engine_register_aeads);603 604void crypto_engine_unregister_aeads(struct aead_engine_alg *algs, int count)605{606 int i;607 608 for (i = count - 1; i >= 0; --i)609 crypto_engine_unregister_aead(&algs[i]);610}611EXPORT_SYMBOL_GPL(crypto_engine_unregister_aeads);612 613int crypto_engine_register_ahash(struct ahash_engine_alg *alg)614{615 if (!alg->op.do_one_request)616 return -EINVAL;617 618 alg->base.halg.base.cra_flags |= CRYPTO_ALG_ENGINE;619 620 return crypto_register_ahash(&alg->base);621}622EXPORT_SYMBOL_GPL(crypto_engine_register_ahash);623 624void crypto_engine_unregister_ahash(struct ahash_engine_alg *alg)625{626 crypto_unregister_ahash(&alg->base);627}628EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahash);629 630int crypto_engine_register_ahashes(struct ahash_engine_alg *algs, int count)631{632 int i, ret;633 634 for (i = 0; i < count; i++) {635 ret = crypto_engine_register_ahash(&algs[i]);636 if (ret)637 goto err;638 }639 640 return 0;641 642err:643 crypto_engine_unregister_ahashes(algs, i);644 645 return ret;646}647EXPORT_SYMBOL_GPL(crypto_engine_register_ahashes);648 649void crypto_engine_unregister_ahashes(struct ahash_engine_alg *algs,650 int count)651{652 int i;653 654 for (i = count - 1; i >= 0; --i)655 crypto_engine_unregister_ahash(&algs[i]);656}657EXPORT_SYMBOL_GPL(crypto_engine_unregister_ahashes);658 659int crypto_engine_register_akcipher(struct akcipher_engine_alg *alg)660{661 if (!alg->op.do_one_request)662 return -EINVAL;663 664 alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;665 666 return crypto_register_akcipher(&alg->base);667}668EXPORT_SYMBOL_GPL(crypto_engine_register_akcipher);669 670void crypto_engine_unregister_akcipher(struct akcipher_engine_alg *alg)671{672 crypto_unregister_akcipher(&alg->base);673}674EXPORT_SYMBOL_GPL(crypto_engine_unregister_akcipher);675 676int crypto_engine_register_kpp(struct kpp_engine_alg *alg)677{678 if (!alg->op.do_one_request)679 return -EINVAL;680 681 alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;682 683 return crypto_register_kpp(&alg->base);684}685EXPORT_SYMBOL_GPL(crypto_engine_register_kpp);686 687void crypto_engine_unregister_kpp(struct kpp_engine_alg *alg)688{689 crypto_unregister_kpp(&alg->base);690}691EXPORT_SYMBOL_GPL(crypto_engine_unregister_kpp);692 693int crypto_engine_register_skcipher(struct skcipher_engine_alg *alg)694{695 if (!alg->op.do_one_request)696 return -EINVAL;697 698 alg->base.base.cra_flags |= CRYPTO_ALG_ENGINE;699 700 return crypto_register_skcipher(&alg->base);701}702EXPORT_SYMBOL_GPL(crypto_engine_register_skcipher);703 704void crypto_engine_unregister_skcipher(struct skcipher_engine_alg *alg)705{706 return crypto_unregister_skcipher(&alg->base);707}708EXPORT_SYMBOL_GPL(crypto_engine_unregister_skcipher);709 710int crypto_engine_register_skciphers(struct skcipher_engine_alg *algs,711 int count)712{713 int i, ret;714 715 for (i = 0; i < count; i++) {716 ret = crypto_engine_register_skcipher(&algs[i]);717 if (ret)718 goto err;719 }720 721 return 0;722 723err:724 crypto_engine_unregister_skciphers(algs, i);725 726 return ret;727}728EXPORT_SYMBOL_GPL(crypto_engine_register_skciphers);729 730void crypto_engine_unregister_skciphers(struct skcipher_engine_alg *algs,731 int count)732{733 int i;734 735 for (i = count - 1; i >= 0; --i)736 crypto_engine_unregister_skcipher(&algs[i]);737}738EXPORT_SYMBOL_GPL(crypto_engine_unregister_skciphers);739 740MODULE_LICENSE("GPL");741MODULE_DESCRIPTION("Crypto hardware engine framework");742