594 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * algif_aead: User-space interface for AEAD algorithms4 *5 * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>6 *7 * This file provides the user-space API for AEAD ciphers.8 *9 * The following concept of the memory management is used:10 *11 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is12 * filled by user space with the data submitted via sendmsg (maybe with13 * MSG_SPLICE_PAGES). Filling up the TX SGL does not cause a crypto operation14 * -- the data will only be tracked by the kernel. Upon receipt of one recvmsg15 * call, the caller must provide a buffer which is tracked with the RX SGL.16 *17 * During the processing of the recvmsg operation, the cipher request is18 * allocated and prepared. As part of the recvmsg operation, the processed19 * TX buffers are extracted from the TX SGL into a separate SGL.20 *21 * After the completion of the crypto operation, the RX SGL and the cipher22 * request is released. The extracted TX SGL parts are released together with23 * the RX SGL release.24 */25 26#include <crypto/internal/aead.h>27#include <crypto/scatterwalk.h>28#include <crypto/if_alg.h>29#include <crypto/skcipher.h>30#include <crypto/null.h>31#include <linux/init.h>32#include <linux/list.h>33#include <linux/kernel.h>34#include <linux/mm.h>35#include <linux/module.h>36#include <linux/net.h>37#include <net/sock.h>38 39struct aead_tfm {40 struct crypto_aead *aead;41 struct crypto_sync_skcipher *null_tfm;42};43 44static inline bool aead_sufficient_data(struct sock *sk)45{46 struct alg_sock *ask = alg_sk(sk);47 struct sock *psk = ask->parent;48 struct alg_sock *pask = alg_sk(psk);49 struct af_alg_ctx *ctx = ask->private;50 struct aead_tfm *aeadc = pask->private;51 struct crypto_aead *tfm = aeadc->aead;52 unsigned int as = crypto_aead_authsize(tfm);53 54 /*55 * The minimum amount of memory needed for an AEAD cipher is56 * the AAD and in case of decryption the tag.57 */58 return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);59}60 61static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)62{63 struct sock *sk = sock->sk;64 struct alg_sock *ask = alg_sk(sk);65 struct sock *psk = ask->parent;66 struct alg_sock *pask = alg_sk(psk);67 struct aead_tfm *aeadc = pask->private;68 struct crypto_aead *tfm = aeadc->aead;69 unsigned int ivsize = crypto_aead_ivsize(tfm);70 71 return af_alg_sendmsg(sock, msg, size, ivsize);72}73 74static int crypto_aead_copy_sgl(struct crypto_sync_skcipher *null_tfm,75 struct scatterlist *src,76 struct scatterlist *dst, unsigned int len)77{78 SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);79 80 skcipher_request_set_sync_tfm(skreq, null_tfm);81 skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP,82 NULL, NULL);83 skcipher_request_set_crypt(skreq, src, dst, len, NULL);84 85 return crypto_skcipher_encrypt(skreq);86}87 88static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,89 size_t ignored, int flags)90{91 struct sock *sk = sock->sk;92 struct alg_sock *ask = alg_sk(sk);93 struct sock *psk = ask->parent;94 struct alg_sock *pask = alg_sk(psk);95 struct af_alg_ctx *ctx = ask->private;96 struct aead_tfm *aeadc = pask->private;97 struct crypto_aead *tfm = aeadc->aead;98 struct crypto_sync_skcipher *null_tfm = aeadc->null_tfm;99 unsigned int i, as = crypto_aead_authsize(tfm);100 struct af_alg_async_req *areq;101 struct af_alg_tsgl *tsgl, *tmp;102 struct scatterlist *rsgl_src, *tsgl_src = NULL;103 int err = 0;104 size_t used = 0; /* [in] TX bufs to be en/decrypted */105 size_t outlen = 0; /* [out] RX bufs produced by kernel */106 size_t usedpages = 0; /* [in] RX bufs to be used from user */107 size_t processed = 0; /* [in] TX bufs to be consumed */108 109 if (!ctx->init || ctx->more) {110 err = af_alg_wait_for_data(sk, flags, 0);111 if (err)112 return err;113 }114 115 /*116 * Data length provided by caller via sendmsg that has not yet been117 * processed.118 */119 used = ctx->used;120 121 /*122 * Make sure sufficient data is present -- note, the same check is also123 * present in sendmsg. The checks in sendmsg shall provide an124 * information to the data sender that something is wrong, but they are125 * irrelevant to maintain the kernel integrity. We need this check126 * here too in case user space decides to not honor the error message127 * in sendmsg and still call recvmsg. This check here protects the128 * kernel integrity.129 */130 if (!aead_sufficient_data(sk))131 return -EINVAL;132 133 /*134 * Calculate the minimum output buffer size holding the result of the135 * cipher operation. When encrypting data, the receiving buffer is136 * larger by the tag length compared to the input buffer as the137 * encryption operation generates the tag. For decryption, the input138 * buffer provides the tag which is consumed resulting in only the139 * plaintext without a buffer for the tag returned to the caller.140 */141 if (ctx->enc)142 outlen = used + as;143 else144 outlen = used - as;145 146 /*147 * The cipher operation input data is reduced by the associated data148 * length as this data is processed separately later on.149 */150 used -= ctx->aead_assoclen;151 152 /* Allocate cipher request for current operation. */153 areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +154 crypto_aead_reqsize(tfm));155 if (IS_ERR(areq))156 return PTR_ERR(areq);157 158 /* convert iovecs of output buffers into RX SGL */159 err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);160 if (err)161 goto free;162 163 /*164 * Ensure output buffer is sufficiently large. If the caller provides165 * less buffer space, only use the relative required input size. This166 * allows AIO operation where the caller sent all data to be processed167 * and the AIO operation performs the operation on the different chunks168 * of the input data.169 */170 if (usedpages < outlen) {171 size_t less = outlen - usedpages;172 173 if (used < less) {174 err = -EINVAL;175 goto free;176 }177 used -= less;178 outlen -= less;179 }180 181 processed = used + ctx->aead_assoclen;182 list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) {183 for (i = 0; i < tsgl->cur; i++) {184 struct scatterlist *process_sg = tsgl->sg + i;185 186 if (!(process_sg->length) || !sg_page(process_sg))187 continue;188 tsgl_src = process_sg;189 break;190 }191 if (tsgl_src)192 break;193 }194 if (processed && !tsgl_src) {195 err = -EFAULT;196 goto free;197 }198 199 /*200 * Copy of AAD from source to destination201 *202 * The AAD is copied to the destination buffer without change. Even203 * when user space uses an in-place cipher operation, the kernel204 * will copy the data as it does not see whether such in-place operation205 * is initiated.206 *207 * To ensure efficiency, the following implementation ensure that the208 * ciphers are invoked to perform a crypto operation in-place. This209 * is achieved by memory management specified as follows.210 */211 212 /* Use the RX SGL as source (and destination) for crypto op. */213 rsgl_src = areq->first_rsgl.sgl.sgt.sgl;214 215 if (ctx->enc) {216 /*217 * Encryption operation - The in-place cipher operation is218 * achieved by the following operation:219 *220 * TX SGL: AAD || PT221 * | |222 * | copy |223 * v v224 * RX SGL: AAD || PT || Tag225 */226 err = crypto_aead_copy_sgl(null_tfm, tsgl_src,227 areq->first_rsgl.sgl.sgt.sgl,228 processed);229 if (err)230 goto free;231 af_alg_pull_tsgl(sk, processed, NULL, 0);232 } else {233 /*234 * Decryption operation - To achieve an in-place cipher235 * operation, the following SGL structure is used:236 *237 * TX SGL: AAD || CT || Tag238 * | | ^239 * | copy | | Create SGL link.240 * v v |241 * RX SGL: AAD || CT ----+242 */243 244 /* Copy AAD || CT to RX SGL buffer for in-place operation. */245 err = crypto_aead_copy_sgl(null_tfm, tsgl_src,246 areq->first_rsgl.sgl.sgt.sgl,247 outlen);248 if (err)249 goto free;250 251 /* Create TX SGL for tag and chain it to RX SGL. */252 areq->tsgl_entries = af_alg_count_tsgl(sk, processed,253 processed - as);254 if (!areq->tsgl_entries)255 areq->tsgl_entries = 1;256 areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl),257 areq->tsgl_entries),258 GFP_KERNEL);259 if (!areq->tsgl) {260 err = -ENOMEM;261 goto free;262 }263 sg_init_table(areq->tsgl, areq->tsgl_entries);264 265 /* Release TX SGL, except for tag data and reassign tag data. */266 af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as);267 268 /* chain the areq TX SGL holding the tag with RX SGL */269 if (usedpages) {270 /* RX SGL present */271 struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl;272 struct scatterlist *sg = sgl_prev->sgt.sgl;273 274 sg_unmark_end(sg + sgl_prev->sgt.nents - 1);275 sg_chain(sg, sgl_prev->sgt.nents + 1, areq->tsgl);276 } else277 /* no RX SGL present (e.g. authentication only) */278 rsgl_src = areq->tsgl;279 }280 281 /* Initialize the crypto operation */282 aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src,283 areq->first_rsgl.sgl.sgt.sgl, used, ctx->iv);284 aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);285 aead_request_set_tfm(&areq->cra_u.aead_req, tfm);286 287 if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {288 /* AIO operation */289 sock_hold(sk);290 areq->iocb = msg->msg_iocb;291 292 /* Remember output size that will be generated. */293 areq->outlen = outlen;294 295 aead_request_set_callback(&areq->cra_u.aead_req,296 CRYPTO_TFM_REQ_MAY_SLEEP,297 af_alg_async_cb, areq);298 err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :299 crypto_aead_decrypt(&areq->cra_u.aead_req);300 301 /* AIO operation in progress */302 if (err == -EINPROGRESS)303 return -EIOCBQUEUED;304 305 sock_put(sk);306 } else {307 /* Synchronous operation */308 aead_request_set_callback(&areq->cra_u.aead_req,309 CRYPTO_TFM_REQ_MAY_SLEEP |310 CRYPTO_TFM_REQ_MAY_BACKLOG,311 crypto_req_done, &ctx->wait);312 err = crypto_wait_req(ctx->enc ?313 crypto_aead_encrypt(&areq->cra_u.aead_req) :314 crypto_aead_decrypt(&areq->cra_u.aead_req),315 &ctx->wait);316 }317 318 319free:320 af_alg_free_resources(areq);321 322 return err ? err : outlen;323}324 325static int aead_recvmsg(struct socket *sock, struct msghdr *msg,326 size_t ignored, int flags)327{328 struct sock *sk = sock->sk;329 int ret = 0;330 331 lock_sock(sk);332 while (msg_data_left(msg)) {333 int err = _aead_recvmsg(sock, msg, ignored, flags);334 335 /*336 * This error covers -EIOCBQUEUED which implies that we can337 * only handle one AIO request. If the caller wants to have338 * multiple AIO requests in parallel, he must make multiple339 * separate AIO calls.340 *341 * Also return the error if no data has been processed so far.342 */343 if (err <= 0) {344 if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)345 ret = err;346 goto out;347 }348 349 ret += err;350 }351 352out:353 af_alg_wmem_wakeup(sk);354 release_sock(sk);355 return ret;356}357 358static struct proto_ops algif_aead_ops = {359 .family = PF_ALG,360 361 .connect = sock_no_connect,362 .socketpair = sock_no_socketpair,363 .getname = sock_no_getname,364 .ioctl = sock_no_ioctl,365 .listen = sock_no_listen,366 .shutdown = sock_no_shutdown,367 .mmap = sock_no_mmap,368 .bind = sock_no_bind,369 .accept = sock_no_accept,370 371 .release = af_alg_release,372 .sendmsg = aead_sendmsg,373 .recvmsg = aead_recvmsg,374 .poll = af_alg_poll,375};376 377static int aead_check_key(struct socket *sock)378{379 int err = 0;380 struct sock *psk;381 struct alg_sock *pask;382 struct aead_tfm *tfm;383 struct sock *sk = sock->sk;384 struct alg_sock *ask = alg_sk(sk);385 386 lock_sock(sk);387 if (!atomic_read(&ask->nokey_refcnt))388 goto unlock_child;389 390 psk = ask->parent;391 pask = alg_sk(ask->parent);392 tfm = pask->private;393 394 err = -ENOKEY;395 lock_sock_nested(psk, SINGLE_DEPTH_NESTING);396 if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)397 goto unlock;398 399 atomic_dec(&pask->nokey_refcnt);400 atomic_set(&ask->nokey_refcnt, 0);401 402 err = 0;403 404unlock:405 release_sock(psk);406unlock_child:407 release_sock(sk);408 409 return err;410}411 412static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,413 size_t size)414{415 int err;416 417 err = aead_check_key(sock);418 if (err)419 return err;420 421 return aead_sendmsg(sock, msg, size);422}423 424static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,425 size_t ignored, int flags)426{427 int err;428 429 err = aead_check_key(sock);430 if (err)431 return err;432 433 return aead_recvmsg(sock, msg, ignored, flags);434}435 436static struct proto_ops algif_aead_ops_nokey = {437 .family = PF_ALG,438 439 .connect = sock_no_connect,440 .socketpair = sock_no_socketpair,441 .getname = sock_no_getname,442 .ioctl = sock_no_ioctl,443 .listen = sock_no_listen,444 .shutdown = sock_no_shutdown,445 .mmap = sock_no_mmap,446 .bind = sock_no_bind,447 .accept = sock_no_accept,448 449 .release = af_alg_release,450 .sendmsg = aead_sendmsg_nokey,451 .recvmsg = aead_recvmsg_nokey,452 .poll = af_alg_poll,453};454 455static void *aead_bind(const char *name, u32 type, u32 mask)456{457 struct aead_tfm *tfm;458 struct crypto_aead *aead;459 struct crypto_sync_skcipher *null_tfm;460 461 tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);462 if (!tfm)463 return ERR_PTR(-ENOMEM);464 465 aead = crypto_alloc_aead(name, type, mask);466 if (IS_ERR(aead)) {467 kfree(tfm);468 return ERR_CAST(aead);469 }470 471 null_tfm = crypto_get_default_null_skcipher();472 if (IS_ERR(null_tfm)) {473 crypto_free_aead(aead);474 kfree(tfm);475 return ERR_CAST(null_tfm);476 }477 478 tfm->aead = aead;479 tfm->null_tfm = null_tfm;480 481 return tfm;482}483 484static void aead_release(void *private)485{486 struct aead_tfm *tfm = private;487 488 crypto_free_aead(tfm->aead);489 crypto_put_default_null_skcipher();490 kfree(tfm);491}492 493static int aead_setauthsize(void *private, unsigned int authsize)494{495 struct aead_tfm *tfm = private;496 497 return crypto_aead_setauthsize(tfm->aead, authsize);498}499 500static int aead_setkey(void *private, const u8 *key, unsigned int keylen)501{502 struct aead_tfm *tfm = private;503 504 return crypto_aead_setkey(tfm->aead, key, keylen);505}506 507static void aead_sock_destruct(struct sock *sk)508{509 struct alg_sock *ask = alg_sk(sk);510 struct af_alg_ctx *ctx = ask->private;511 struct sock *psk = ask->parent;512 struct alg_sock *pask = alg_sk(psk);513 struct aead_tfm *aeadc = pask->private;514 struct crypto_aead *tfm = aeadc->aead;515 unsigned int ivlen = crypto_aead_ivsize(tfm);516 517 af_alg_pull_tsgl(sk, ctx->used, NULL, 0);518 sock_kzfree_s(sk, ctx->iv, ivlen);519 sock_kfree_s(sk, ctx, ctx->len);520 af_alg_release_parent(sk);521}522 523static int aead_accept_parent_nokey(void *private, struct sock *sk)524{525 struct af_alg_ctx *ctx;526 struct alg_sock *ask = alg_sk(sk);527 struct aead_tfm *tfm = private;528 struct crypto_aead *aead = tfm->aead;529 unsigned int len = sizeof(*ctx);530 unsigned int ivlen = crypto_aead_ivsize(aead);531 532 ctx = sock_kmalloc(sk, len, GFP_KERNEL);533 if (!ctx)534 return -ENOMEM;535 memset(ctx, 0, len);536 537 ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);538 if (!ctx->iv) {539 sock_kfree_s(sk, ctx, len);540 return -ENOMEM;541 }542 memset(ctx->iv, 0, ivlen);543 544 INIT_LIST_HEAD(&ctx->tsgl_list);545 ctx->len = len;546 crypto_init_wait(&ctx->wait);547 548 ask->private = ctx;549 550 sk->sk_destruct = aead_sock_destruct;551 552 return 0;553}554 555static int aead_accept_parent(void *private, struct sock *sk)556{557 struct aead_tfm *tfm = private;558 559 if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)560 return -ENOKEY;561 562 return aead_accept_parent_nokey(private, sk);563}564 565static const struct af_alg_type algif_type_aead = {566 .bind = aead_bind,567 .release = aead_release,568 .setkey = aead_setkey,569 .setauthsize = aead_setauthsize,570 .accept = aead_accept_parent,571 .accept_nokey = aead_accept_parent_nokey,572 .ops = &algif_aead_ops,573 .ops_nokey = &algif_aead_ops_nokey,574 .name = "aead",575 .owner = THIS_MODULE576};577 578static int __init algif_aead_init(void)579{580 return af_alg_register_type(&algif_type_aead);581}582 583static void __exit algif_aead_exit(void)584{585 int err = af_alg_unregister_type(&algif_type_aead);586 BUG_ON(err);587}588 589module_init(algif_aead_init);590module_exit(algif_aead_exit);591MODULE_LICENSE("GPL");592MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");593MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");594