618 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Code related to the io_uring_register() syscall4 *5 * Copyright (C) 2023 Jens Axboe6 */7#include <linux/kernel.h>8#include <linux/errno.h>9#include <linux/syscalls.h>10#include <linux/refcount.h>11#include <linux/bits.h>12#include <linux/fs.h>13#include <linux/file.h>14#include <linux/slab.h>15#include <linux/uaccess.h>16#include <linux/nospec.h>17#include <linux/compat.h>18#include <linux/io_uring.h>19#include <linux/io_uring_types.h>20 21#include "io_uring.h"22#include "opdef.h"23#include "tctx.h"24#include "rsrc.h"25#include "sqpoll.h"26#include "register.h"27#include "cancel.h"28#include "kbuf.h"29#include "napi.h"30#include "eventfd.h"31 32#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \33 IORING_REGISTER_LAST + IORING_OP_LAST)34 35static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,36 unsigned nr_args)37{38 struct io_uring_probe *p;39 size_t size;40 int i, ret;41 42 if (nr_args > IORING_OP_LAST)43 nr_args = IORING_OP_LAST;44 45 size = struct_size(p, ops, nr_args);46 p = kzalloc(size, GFP_KERNEL);47 if (!p)48 return -ENOMEM;49 50 ret = -EFAULT;51 if (copy_from_user(p, arg, size))52 goto out;53 ret = -EINVAL;54 if (memchr_inv(p, 0, size))55 goto out;56 57 p->last_op = IORING_OP_LAST - 1;58 59 for (i = 0; i < nr_args; i++) {60 p->ops[i].op = i;61 if (io_uring_op_supported(i))62 p->ops[i].flags = IO_URING_OP_SUPPORTED;63 }64 p->ops_len = i;65 66 ret = 0;67 if (copy_to_user(arg, p, size))68 ret = -EFAULT;69out:70 kfree(p);71 return ret;72}73 74int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)75{76 const struct cred *creds;77 78 creds = xa_erase(&ctx->personalities, id);79 if (creds) {80 put_cred(creds);81 return 0;82 }83 84 return -EINVAL;85}86 87 88static int io_register_personality(struct io_ring_ctx *ctx)89{90 const struct cred *creds;91 u32 id;92 int ret;93 94 creds = get_current_cred();95 96 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,97 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);98 if (ret < 0) {99 put_cred(creds);100 return ret;101 }102 return id;103}104 105static __cold int io_register_restrictions(struct io_ring_ctx *ctx,106 void __user *arg, unsigned int nr_args)107{108 struct io_uring_restriction *res;109 size_t size;110 int i, ret;111 112 /* Restrictions allowed only if rings started disabled */113 if (!(ctx->flags & IORING_SETUP_R_DISABLED))114 return -EBADFD;115 116 /* We allow only a single restrictions registration */117 if (ctx->restrictions.registered)118 return -EBUSY;119 120 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)121 return -EINVAL;122 123 size = array_size(nr_args, sizeof(*res));124 if (size == SIZE_MAX)125 return -EOVERFLOW;126 127 res = memdup_user(arg, size);128 if (IS_ERR(res))129 return PTR_ERR(res);130 131 ret = 0;132 133 for (i = 0; i < nr_args; i++) {134 switch (res[i].opcode) {135 case IORING_RESTRICTION_REGISTER_OP:136 if (res[i].register_op >= IORING_REGISTER_LAST) {137 ret = -EINVAL;138 goto out;139 }140 141 __set_bit(res[i].register_op,142 ctx->restrictions.register_op);143 break;144 case IORING_RESTRICTION_SQE_OP:145 if (res[i].sqe_op >= IORING_OP_LAST) {146 ret = -EINVAL;147 goto out;148 }149 150 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);151 break;152 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:153 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;154 break;155 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:156 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;157 break;158 default:159 ret = -EINVAL;160 goto out;161 }162 }163 164out:165 /* Reset all restrictions if an error happened */166 if (ret != 0)167 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));168 else169 ctx->restrictions.registered = true;170 171 kfree(res);172 return ret;173}174 175static HWJS_SUSPENDS int io_register_enable_rings(struct io_ring_ctx *ctx)176{177 if (!(ctx->flags & IORING_SETUP_R_DISABLED))178 return -EBADFD;179 180 if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) {181 WRITE_ONCE(ctx->submitter_task, get_task_struct(current));182 /*183 * Lazy activation attempts would fail if it was polled before184 * submitter_task is set.185 */186 if (wq_has_sleeper(&ctx->poll_wq))187 io_activate_pollwq(ctx);188 }189 190 if (ctx->restrictions.registered)191 ctx->restricted = 1;192 193 ctx->flags &= ~IORING_SETUP_R_DISABLED;194 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))195 wake_up(&ctx->sq_data->wait);196 return 0;197}198 199static HWJS_SUSPENDS __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,200 cpumask_var_t new_mask)201{202 int ret;203 204 if (!(ctx->flags & IORING_SETUP_SQPOLL)) {205 ret = io_wq_cpu_affinity(current->io_uring, new_mask);206 } else {207 mutex_unlock(&ctx->uring_lock);208 ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);209 mutex_lock(&ctx->uring_lock);210 }211 212 return ret;213}214 215static HWJS_SUSPENDS __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,216 void __user *arg, unsigned len)217{218 cpumask_var_t new_mask;219 int ret;220 221 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))222 return -ENOMEM;223 224 cpumask_clear(new_mask);225 if (len > cpumask_size())226 len = cpumask_size();227 228#ifdef CONFIG_COMPAT229 if (in_compat_syscall())230 ret = compat_get_bitmap(cpumask_bits(new_mask),231 (const compat_ulong_t __user *)arg,232 len * 8 /* CHAR_BIT */);233 else234#endif235 ret = copy_from_user(new_mask, arg, len);236 237 if (ret) {238 free_cpumask_var(new_mask);239 return -EFAULT;240 }241 242 ret = __io_register_iowq_aff(ctx, new_mask);243 free_cpumask_var(new_mask);244 return ret;245}246 247static HWJS_SUSPENDS __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx)248{249 return __io_register_iowq_aff(ctx, NULL);250}251 252static HWJS_SUSPENDS __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,253 void __user *arg)254 __must_hold(&ctx->uring_lock)255{256 struct io_tctx_node *node;257 struct io_uring_task *tctx = NULL;258 struct io_sq_data *sqd = NULL;259 __u32 new_count[2];260 int i, ret;261 262 if (copy_from_user(new_count, arg, sizeof(new_count)))263 return -EFAULT;264 for (i = 0; i < ARRAY_SIZE(new_count); i++)265 if (new_count[i] > INT_MAX)266 return -EINVAL;267 268 if (ctx->flags & IORING_SETUP_SQPOLL) {269 sqd = ctx->sq_data;270 if (sqd) {271 /*272 * Observe the correct sqd->lock -> ctx->uring_lock273 * ordering. Fine to drop uring_lock here, we hold274 * a ref to the ctx.275 */276 refcount_inc(&sqd->refs);277 mutex_unlock(&ctx->uring_lock);278 mutex_lock(&sqd->lock);279 mutex_lock(&ctx->uring_lock);280 if (sqd->thread)281 tctx = sqd->thread->io_uring;282 }283 } else {284 tctx = current->io_uring;285 }286 287 BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));288 289 for (i = 0; i < ARRAY_SIZE(new_count); i++)290 if (new_count[i])291 ctx->iowq_limits[i] = new_count[i];292 ctx->iowq_limits_set = true;293 294 if (tctx && tctx->io_wq) {295 ret = io_wq_max_workers(tctx->io_wq, new_count);296 if (ret)297 goto err;298 } else {299 memset(new_count, 0, sizeof(new_count));300 }301 302 if (sqd) {303 mutex_unlock(&ctx->uring_lock);304 mutex_unlock(&sqd->lock);305 io_put_sq_data(sqd);306 mutex_lock(&ctx->uring_lock);307 }308 309 if (copy_to_user(arg, new_count, sizeof(new_count)))310 return -EFAULT;311 312 /* that's it for SQPOLL, only the SQPOLL task creates requests */313 if (sqd)314 return 0;315 316 /* now propagate the restriction to all registered users */317 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {318 tctx = node->task->io_uring;319 if (WARN_ON_ONCE(!tctx->io_wq))320 continue;321 322 for (i = 0; i < ARRAY_SIZE(new_count); i++)323 new_count[i] = ctx->iowq_limits[i];324 /* ignore errors, it always returns zero anyway */325 (void)io_wq_max_workers(tctx->io_wq, new_count);326 }327 return 0;328err:329 if (sqd) {330 mutex_unlock(&ctx->uring_lock);331 mutex_unlock(&sqd->lock);332 io_put_sq_data(sqd);333 mutex_lock(&ctx->uring_lock);334 }335 return ret;336}337 338static int io_register_clock(struct io_ring_ctx *ctx,339 struct io_uring_clock_register __user *arg)340{341 struct io_uring_clock_register reg;342 343 if (copy_from_user(®, arg, sizeof(reg)))344 return -EFAULT;345 if (memchr_inv(®.__resv, 0, sizeof(reg.__resv)))346 return -EINVAL;347 348 switch (reg.clockid) {349 case CLOCK_MONOTONIC:350 ctx->clock_offset = 0;351 break;352 case CLOCK_BOOTTIME:353 ctx->clock_offset = TK_OFFS_BOOT;354 break;355 default:356 return -EINVAL;357 }358 359 ctx->clockid = reg.clockid;360 return 0;361}362 363static HWJS_SUSPENDS int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,364 void __user *arg, unsigned nr_args)365 __releases(ctx->uring_lock)366 __acquires(ctx->uring_lock)367{368 int ret;369 370 /*371 * We don't quiesce the refs for register anymore and so it can't be372 * dying as we're holding a file ref here.373 */374 if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))375 return -ENXIO;376 377 if (ctx->submitter_task && ctx->submitter_task != current)378 return -EEXIST;379 380 if (ctx->restricted) {381 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);382 if (!test_bit(opcode, ctx->restrictions.register_op))383 return -EACCES;384 }385 386 switch (opcode) {387 case IORING_REGISTER_BUFFERS:388 ret = -EFAULT;389 if (!arg)390 break;391 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);392 break;393 case IORING_UNREGISTER_BUFFERS:394 ret = -EINVAL;395 if (arg || nr_args)396 break;397 ret = io_sqe_buffers_unregister(ctx);398 break;399 case IORING_REGISTER_FILES:400 ret = -EFAULT;401 if (!arg)402 break;403 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);404 break;405 case IORING_UNREGISTER_FILES:406 ret = -EINVAL;407 if (arg || nr_args)408 break;409 ret = io_sqe_files_unregister(ctx);410 break;411 case IORING_REGISTER_FILES_UPDATE:412 ret = io_register_files_update(ctx, arg, nr_args);413 break;414 case IORING_REGISTER_EVENTFD:415 ret = -EINVAL;416 if (nr_args != 1)417 break;418 ret = io_eventfd_register(ctx, arg, 0);419 break;420 case IORING_REGISTER_EVENTFD_ASYNC:421 ret = -EINVAL;422 if (nr_args != 1)423 break;424 ret = io_eventfd_register(ctx, arg, 1);425 break;426 case IORING_UNREGISTER_EVENTFD:427 ret = -EINVAL;428 if (arg || nr_args)429 break;430 ret = io_eventfd_unregister(ctx);431 break;432 case IORING_REGISTER_PROBE:433 ret = -EINVAL;434 if (!arg || nr_args > 256)435 break;436 ret = io_probe(ctx, arg, nr_args);437 break;438 case IORING_REGISTER_PERSONALITY:439 ret = -EINVAL;440 if (arg || nr_args)441 break;442 ret = io_register_personality(ctx);443 break;444 case IORING_UNREGISTER_PERSONALITY:445 ret = -EINVAL;446 if (arg)447 break;448 ret = io_unregister_personality(ctx, nr_args);449 break;450 case IORING_REGISTER_ENABLE_RINGS:451 ret = -EINVAL;452 if (arg || nr_args)453 break;454 ret = io_register_enable_rings(ctx);455 break;456 case IORING_REGISTER_RESTRICTIONS:457 ret = io_register_restrictions(ctx, arg, nr_args);458 break;459 case IORING_REGISTER_FILES2:460 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);461 break;462 case IORING_REGISTER_FILES_UPDATE2:463 ret = io_register_rsrc_update(ctx, arg, nr_args,464 IORING_RSRC_FILE);465 break;466 case IORING_REGISTER_BUFFERS2:467 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);468 break;469 case IORING_REGISTER_BUFFERS_UPDATE:470 ret = io_register_rsrc_update(ctx, arg, nr_args,471 IORING_RSRC_BUFFER);472 break;473 case IORING_REGISTER_IOWQ_AFF:474 ret = -EINVAL;475 if (!arg || !nr_args)476 break;477 ret = io_register_iowq_aff(ctx, arg, nr_args);478 break;479 case IORING_UNREGISTER_IOWQ_AFF:480 ret = -EINVAL;481 if (arg || nr_args)482 break;483 ret = io_unregister_iowq_aff(ctx);484 break;485 case IORING_REGISTER_IOWQ_MAX_WORKERS:486 ret = -EINVAL;487 if (!arg || nr_args != 2)488 break;489 ret = io_register_iowq_max_workers(ctx, arg);490 break;491 case IORING_REGISTER_RING_FDS:492 ret = io_ringfd_register(ctx, arg, nr_args);493 break;494 case IORING_UNREGISTER_RING_FDS:495 ret = io_ringfd_unregister(ctx, arg, nr_args);496 break;497 case IORING_REGISTER_PBUF_RING:498 ret = -EINVAL;499 if (!arg || nr_args != 1)500 break;501 ret = io_register_pbuf_ring(ctx, arg);502 break;503 case IORING_UNREGISTER_PBUF_RING:504 ret = -EINVAL;505 if (!arg || nr_args != 1)506 break;507 ret = io_unregister_pbuf_ring(ctx, arg);508 break;509 case IORING_REGISTER_SYNC_CANCEL:510 ret = -EINVAL;511 if (!arg || nr_args != 1)512 break;513 ret = io_sync_cancel(ctx, arg);514 break;515 case IORING_REGISTER_FILE_ALLOC_RANGE:516 ret = -EINVAL;517 if (!arg || nr_args)518 break;519 ret = io_register_file_alloc_range(ctx, arg);520 break;521 case IORING_REGISTER_PBUF_STATUS:522 ret = -EINVAL;523 if (!arg || nr_args != 1)524 break;525 ret = io_register_pbuf_status(ctx, arg);526 break;527 case IORING_REGISTER_NAPI:528 ret = -EINVAL;529 if (!arg || nr_args != 1)530 break;531 ret = io_register_napi(ctx, arg);532 break;533 case IORING_UNREGISTER_NAPI:534 ret = -EINVAL;535 if (nr_args != 1)536 break;537 ret = io_unregister_napi(ctx, arg);538 break;539 case IORING_REGISTER_CLOCK:540 ret = -EINVAL;541 if (!arg || nr_args)542 break;543 ret = io_register_clock(ctx, arg);544 break;545 case IORING_REGISTER_CLONE_BUFFERS:546 ret = -EINVAL;547 if (!arg || nr_args != 1)548 break;549 ret = io_register_clone_buffers(ctx, arg);550 break;551 default:552 ret = -EINVAL;553 break;554 }555 556 return ret;557}558 559/*560 * Given an 'fd' value, return the ctx associated with if. If 'registered' is561 * true, then the registered index is used. Otherwise, the normal fd table.562 * Caller must call fput() on the returned file, unless it's an ERR_PTR.563 */564struct file *io_uring_register_get_file(unsigned int fd, bool registered)565{566 struct file *file;567 568 if (registered) {569 /*570 * Ring fd has been registered via IORING_REGISTER_RING_FDS, we571 * need only dereference our task private array to find it.572 */573 struct io_uring_task *tctx = current->io_uring;574 575 if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))576 return ERR_PTR(-EINVAL);577 fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);578 file = tctx->registered_rings[fd];579 } else {580 file = fget(fd);581 }582 583 if (unlikely(!file))584 return ERR_PTR(-EBADF);585 if (io_is_uring_fops(file))586 return file;587 fput(file);588 return ERR_PTR(-EOPNOTSUPP);589}590 591SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,592 void __user *, arg, unsigned int, nr_args)593{594 struct io_ring_ctx *ctx;595 long ret = -EBADF;596 struct file *file;597 bool use_registered_ring;598 599 use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING);600 opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;601 602 if (opcode >= IORING_REGISTER_LAST)603 return -EINVAL;604 605 file = io_uring_register_get_file(fd, use_registered_ring);606 if (IS_ERR(file))607 return PTR_ERR(file);608 ctx = file->private_data;609 610 mutex_lock(&ctx->uring_lock);611 ret = __io_uring_register(ctx, opcode, arg, nr_args);612 mutex_unlock(&ctx->uring_lock);613 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);614 if (!use_registered_ring)615 fput(file);616 return ret;617}618