776 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * io_uring opcode handling table4 */5#include <linux/kernel.h>6#include <linux/errno.h>7#include <linux/fs.h>8#include <linux/file.h>9#include <linux/io_uring.h>10 11#include "io_uring.h"12#include "opdef.h"13#include "refs.h"14#include "tctx.h"15#include "sqpoll.h"16#include "fdinfo.h"17#include "kbuf.h"18#include "rsrc.h"19 20#include "xattr.h"21#include "nop.h"22#include "fs.h"23#include "splice.h"24#include "sync.h"25#include "advise.h"26#include "openclose.h"27#include "uring_cmd.h"28#include "epoll.h"29#include "statx.h"30#include "net.h"31#include "msg_ring.h"32#include "timeout.h"33#include "poll.h"34#include "cancel.h"35#include "rw.h"36#include "waitid.h"37#include "futex.h"38#include "truncate.h"39 40static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)41{42 WARN_ON_ONCE(1);43 return -ECANCELED;44}45 46static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,47 const struct io_uring_sqe *sqe)48{49 return -EOPNOTSUPP;50}51 52const struct io_issue_def io_issue_defs[] = {53 [IORING_OP_NOP] = {54 .audit_skip = 1,55 .iopoll = 1,56 .prep = io_nop_prep,57 .issue = io_nop,58 },59 [IORING_OP_READV] = {60 .needs_file = 1,61 .unbound_nonreg_file = 1,62 .pollin = 1,63 .buffer_select = 1,64 .plug = 1,65 .audit_skip = 1,66 .ioprio = 1,67 .iopoll = 1,68 .iopoll_queue = 1,69 .vectored = 1,70 .async_size = sizeof(struct io_async_rw),71 .prep = io_prep_readv,72 .issue = io_read,73 },74 [IORING_OP_WRITEV] = {75 .needs_file = 1,76 .hash_reg_file = 1,77 .unbound_nonreg_file = 1,78 .pollout = 1,79 .plug = 1,80 .audit_skip = 1,81 .ioprio = 1,82 .iopoll = 1,83 .iopoll_queue = 1,84 .vectored = 1,85 .async_size = sizeof(struct io_async_rw),86 .prep = io_prep_writev,87 .issue = io_write,88 },89 [IORING_OP_FSYNC] = {90 .needs_file = 1,91 .audit_skip = 1,92 .prep = io_fsync_prep,93 .issue = io_fsync,94 },95 [IORING_OP_READ_FIXED] = {96 .needs_file = 1,97 .unbound_nonreg_file = 1,98 .pollin = 1,99 .plug = 1,100 .audit_skip = 1,101 .ioprio = 1,102 .iopoll = 1,103 .iopoll_queue = 1,104 .async_size = sizeof(struct io_async_rw),105 .prep = io_prep_read_fixed,106 .issue = io_read,107 },108 [IORING_OP_WRITE_FIXED] = {109 .needs_file = 1,110 .hash_reg_file = 1,111 .unbound_nonreg_file = 1,112 .pollout = 1,113 .plug = 1,114 .audit_skip = 1,115 .ioprio = 1,116 .iopoll = 1,117 .iopoll_queue = 1,118 .async_size = sizeof(struct io_async_rw),119 .prep = io_prep_write_fixed,120 .issue = io_write,121 },122 [IORING_OP_POLL_ADD] = {123 .needs_file = 1,124 .unbound_nonreg_file = 1,125 .audit_skip = 1,126 .prep = io_poll_add_prep,127 .issue = io_poll_add,128 },129 [IORING_OP_POLL_REMOVE] = {130 .audit_skip = 1,131 .prep = io_poll_remove_prep,132 .issue = io_poll_remove,133 },134 [IORING_OP_SYNC_FILE_RANGE] = {135 .needs_file = 1,136 .audit_skip = 1,137 .prep = io_sfr_prep,138 .issue = io_sync_file_range,139 },140 [IORING_OP_SENDMSG] = {141 .needs_file = 1,142 .unbound_nonreg_file = 1,143 .pollout = 1,144 .ioprio = 1,145#if defined(CONFIG_NET)146 .async_size = sizeof(struct io_async_msghdr),147 .prep = io_sendmsg_prep,148 .issue = io_sendmsg,149#else150 .prep = io_eopnotsupp_prep,151#endif152 },153 [IORING_OP_RECVMSG] = {154 .needs_file = 1,155 .unbound_nonreg_file = 1,156 .pollin = 1,157 .buffer_select = 1,158 .ioprio = 1,159#if defined(CONFIG_NET)160 .async_size = sizeof(struct io_async_msghdr),161 .prep = io_recvmsg_prep,162 .issue = io_recvmsg,163#else164 .prep = io_eopnotsupp_prep,165#endif166 },167 [IORING_OP_TIMEOUT] = {168 .audit_skip = 1,169 .async_size = sizeof(struct io_timeout_data),170 .prep = io_timeout_prep,171 .issue = io_timeout,172 },173 [IORING_OP_TIMEOUT_REMOVE] = {174 /* used by timeout updates' prep() */175 .audit_skip = 1,176 .prep = io_timeout_remove_prep,177 .issue = io_timeout_remove,178 },179 [IORING_OP_ACCEPT] = {180 .needs_file = 1,181 .unbound_nonreg_file = 1,182 .pollin = 1,183 .poll_exclusive = 1,184 .ioprio = 1, /* used for flags */185#if defined(CONFIG_NET)186 .prep = io_accept_prep,187 .issue = io_accept,188#else189 .prep = io_eopnotsupp_prep,190#endif191 },192 [IORING_OP_ASYNC_CANCEL] = {193 .audit_skip = 1,194 .prep = io_async_cancel_prep,195 .issue = io_async_cancel,196 },197 [IORING_OP_LINK_TIMEOUT] = {198 .audit_skip = 1,199 .async_size = sizeof(struct io_timeout_data),200 .prep = io_link_timeout_prep,201 .issue = io_no_issue,202 },203 [IORING_OP_CONNECT] = {204 .needs_file = 1,205 .unbound_nonreg_file = 1,206 .pollout = 1,207#if defined(CONFIG_NET)208 .async_size = sizeof(struct io_async_msghdr),209 .prep = io_connect_prep,210 .issue = io_connect,211#else212 .prep = io_eopnotsupp_prep,213#endif214 },215 [IORING_OP_FALLOCATE] = {216 .needs_file = 1,217 .prep = io_fallocate_prep,218 .issue = io_fallocate,219 },220 [IORING_OP_OPENAT] = {221 .prep = io_openat_prep,222 .issue = io_openat,223 },224 [IORING_OP_CLOSE] = {225 .prep = io_close_prep,226 .issue = io_close,227 },228 [IORING_OP_FILES_UPDATE] = {229 .audit_skip = 1,230 .iopoll = 1,231 .prep = io_files_update_prep,232 .issue = io_files_update,233 },234 [IORING_OP_STATX] = {235 .audit_skip = 1,236 .prep = io_statx_prep,237 .issue = io_statx,238 },239 [IORING_OP_READ] = {240 .needs_file = 1,241 .unbound_nonreg_file = 1,242 .pollin = 1,243 .buffer_select = 1,244 .plug = 1,245 .audit_skip = 1,246 .ioprio = 1,247 .iopoll = 1,248 .iopoll_queue = 1,249 .async_size = sizeof(struct io_async_rw),250 .prep = io_prep_read,251 .issue = io_read,252 },253 [IORING_OP_WRITE] = {254 .needs_file = 1,255 .hash_reg_file = 1,256 .unbound_nonreg_file = 1,257 .pollout = 1,258 .plug = 1,259 .audit_skip = 1,260 .ioprio = 1,261 .iopoll = 1,262 .iopoll_queue = 1,263 .async_size = sizeof(struct io_async_rw),264 .prep = io_prep_write,265 .issue = io_write,266 },267 [IORING_OP_FADVISE] = {268 .needs_file = 1,269 .audit_skip = 1,270 .prep = io_fadvise_prep,271 .issue = io_fadvise,272 },273 [IORING_OP_MADVISE] = {274 .audit_skip = 1,275 .prep = io_madvise_prep,276 .issue = io_madvise,277 },278 [IORING_OP_SEND] = {279 .needs_file = 1,280 .unbound_nonreg_file = 1,281 .pollout = 1,282 .audit_skip = 1,283 .ioprio = 1,284 .buffer_select = 1,285#if defined(CONFIG_NET)286 .async_size = sizeof(struct io_async_msghdr),287 .prep = io_sendmsg_prep,288 .issue = io_send,289#else290 .prep = io_eopnotsupp_prep,291#endif292 },293 [IORING_OP_RECV] = {294 .needs_file = 1,295 .unbound_nonreg_file = 1,296 .pollin = 1,297 .buffer_select = 1,298 .audit_skip = 1,299 .ioprio = 1,300#if defined(CONFIG_NET)301 .async_size = sizeof(struct io_async_msghdr),302 .prep = io_recvmsg_prep,303 .issue = io_recv,304#else305 .prep = io_eopnotsupp_prep,306#endif307 },308 [IORING_OP_OPENAT2] = {309 .prep = io_openat2_prep,310 .issue = io_openat2,311 },312 [IORING_OP_EPOLL_CTL] = {313 .unbound_nonreg_file = 1,314 .audit_skip = 1,315#if defined(CONFIG_EPOLL)316 .prep = io_epoll_ctl_prep,317 .issue = io_epoll_ctl,318#else319 .prep = io_eopnotsupp_prep,320#endif321 },322 [IORING_OP_SPLICE] = {323 .needs_file = 1,324 .hash_reg_file = 1,325 .unbound_nonreg_file = 1,326 .audit_skip = 1,327 .prep = io_splice_prep,328 .issue = io_splice,329 },330 [IORING_OP_PROVIDE_BUFFERS] = {331 .audit_skip = 1,332 .iopoll = 1,333 .prep = io_provide_buffers_prep,334 .issue = io_provide_buffers,335 },336 [IORING_OP_REMOVE_BUFFERS] = {337 .audit_skip = 1,338 .iopoll = 1,339 .prep = io_remove_buffers_prep,340 .issue = io_remove_buffers,341 },342 [IORING_OP_TEE] = {343 .needs_file = 1,344 .hash_reg_file = 1,345 .unbound_nonreg_file = 1,346 .audit_skip = 1,347 .prep = io_tee_prep,348 .issue = io_tee,349 },350 [IORING_OP_SHUTDOWN] = {351 .needs_file = 1,352#if defined(CONFIG_NET)353 .prep = io_shutdown_prep,354 .issue = io_shutdown,355#else356 .prep = io_eopnotsupp_prep,357#endif358 },359 [IORING_OP_RENAMEAT] = {360 .prep = io_renameat_prep,361 .issue = io_renameat,362 },363 [IORING_OP_UNLINKAT] = {364 .prep = io_unlinkat_prep,365 .issue = io_unlinkat,366 },367 [IORING_OP_MKDIRAT] = {368 .prep = io_mkdirat_prep,369 .issue = io_mkdirat,370 },371 [IORING_OP_SYMLINKAT] = {372 .prep = io_symlinkat_prep,373 .issue = io_symlinkat,374 },375 [IORING_OP_LINKAT] = {376 .prep = io_linkat_prep,377 .issue = io_linkat,378 },379 [IORING_OP_MSG_RING] = {380 .needs_file = 1,381 .iopoll = 1,382 .prep = io_msg_ring_prep,383 .issue = io_msg_ring,384 },385 [IORING_OP_FSETXATTR] = {386 .needs_file = 1,387 .prep = io_fsetxattr_prep,388 .issue = io_fsetxattr,389 },390 [IORING_OP_SETXATTR] = {391 .prep = io_setxattr_prep,392 .issue = io_setxattr,393 },394 [IORING_OP_FGETXATTR] = {395 .needs_file = 1,396 .prep = io_fgetxattr_prep,397 .issue = io_fgetxattr,398 },399 [IORING_OP_GETXATTR] = {400 .prep = io_getxattr_prep,401 .issue = io_getxattr,402 },403 [IORING_OP_SOCKET] = {404 .audit_skip = 1,405#if defined(CONFIG_NET)406 .prep = io_socket_prep,407 .issue = io_socket,408#else409 .prep = io_eopnotsupp_prep,410#endif411 },412 [IORING_OP_URING_CMD] = {413 .needs_file = 1,414 .plug = 1,415 .iopoll = 1,416 .iopoll_queue = 1,417 .async_size = 2 * sizeof(struct io_uring_sqe),418 .prep = io_uring_cmd_prep,419 .issue = io_uring_cmd,420 },421 [IORING_OP_SEND_ZC] = {422 .needs_file = 1,423 .unbound_nonreg_file = 1,424 .pollout = 1,425 .audit_skip = 1,426 .ioprio = 1,427#if defined(CONFIG_NET)428 .async_size = sizeof(struct io_async_msghdr),429 .prep = io_send_zc_prep,430 .issue = io_send_zc,431#else432 .prep = io_eopnotsupp_prep,433#endif434 },435 [IORING_OP_SENDMSG_ZC] = {436 .needs_file = 1,437 .unbound_nonreg_file = 1,438 .pollout = 1,439 .ioprio = 1,440#if defined(CONFIG_NET)441 .async_size = sizeof(struct io_async_msghdr),442 .prep = io_send_zc_prep,443 .issue = io_sendmsg_zc,444#else445 .prep = io_eopnotsupp_prep,446#endif447 },448 [IORING_OP_READ_MULTISHOT] = {449 .needs_file = 1,450 .unbound_nonreg_file = 1,451 .pollin = 1,452 .buffer_select = 1,453 .audit_skip = 1,454 .async_size = sizeof(struct io_async_rw),455 .prep = io_read_mshot_prep,456 .issue = io_read_mshot,457 },458 [IORING_OP_WAITID] = {459 .async_size = sizeof(struct io_waitid_async),460 .prep = io_waitid_prep,461 .issue = io_waitid,462 },463 [IORING_OP_FUTEX_WAIT] = {464#if defined(CONFIG_FUTEX)465 .prep = io_futex_prep,466 .issue = io_futex_wait,467#else468 .prep = io_eopnotsupp_prep,469#endif470 },471 [IORING_OP_FUTEX_WAKE] = {472#if defined(CONFIG_FUTEX)473 .prep = io_futex_prep,474 .issue = io_futex_wake,475#else476 .prep = io_eopnotsupp_prep,477#endif478 },479 [IORING_OP_FUTEX_WAITV] = {480#if defined(CONFIG_FUTEX)481 .prep = io_futexv_prep,482 .issue = io_futexv_wait,483#else484 .prep = io_eopnotsupp_prep,485#endif486 },487 [IORING_OP_FIXED_FD_INSTALL] = {488 .needs_file = 1,489 .prep = io_install_fixed_fd_prep,490 .issue = io_install_fixed_fd,491 },492 [IORING_OP_FTRUNCATE] = {493 .needs_file = 1,494 .hash_reg_file = 1,495 .prep = io_ftruncate_prep,496 .issue = io_ftruncate,497 },498 [IORING_OP_BIND] = {499#if defined(CONFIG_NET)500 .needs_file = 1,501 .prep = io_bind_prep,502 .issue = io_bind,503 .async_size = sizeof(struct io_async_msghdr),504#else505 .prep = io_eopnotsupp_prep,506#endif507 },508 [IORING_OP_LISTEN] = {509#if defined(CONFIG_NET)510 .needs_file = 1,511 .prep = io_listen_prep,512 .issue = io_listen,513 .async_size = sizeof(struct io_async_msghdr),514#else515 .prep = io_eopnotsupp_prep,516#endif517 },518};519 520const struct io_cold_def io_cold_defs[] = {521 [IORING_OP_NOP] = {522 .name = "NOP",523 },524 [IORING_OP_READV] = {525 .name = "READV",526 .cleanup = io_readv_writev_cleanup,527 .fail = io_rw_fail,528 },529 [IORING_OP_WRITEV] = {530 .name = "WRITEV",531 .cleanup = io_readv_writev_cleanup,532 .fail = io_rw_fail,533 },534 [IORING_OP_FSYNC] = {535 .name = "FSYNC",536 },537 [IORING_OP_READ_FIXED] = {538 .name = "READ_FIXED",539 .cleanup = io_readv_writev_cleanup,540 .fail = io_rw_fail,541 },542 [IORING_OP_WRITE_FIXED] = {543 .name = "WRITE_FIXED",544 .cleanup = io_readv_writev_cleanup,545 .fail = io_rw_fail,546 },547 [IORING_OP_POLL_ADD] = {548 .name = "POLL_ADD",549 },550 [IORING_OP_POLL_REMOVE] = {551 .name = "POLL_REMOVE",552 },553 [IORING_OP_SYNC_FILE_RANGE] = {554 .name = "SYNC_FILE_RANGE",555 },556 [IORING_OP_SENDMSG] = {557 .name = "SENDMSG",558#if defined(CONFIG_NET)559 .cleanup = io_sendmsg_recvmsg_cleanup,560 .fail = io_sendrecv_fail,561#endif562 },563 [IORING_OP_RECVMSG] = {564 .name = "RECVMSG",565#if defined(CONFIG_NET)566 .cleanup = io_sendmsg_recvmsg_cleanup,567 .fail = io_sendrecv_fail,568#endif569 },570 [IORING_OP_TIMEOUT] = {571 .name = "TIMEOUT",572 },573 [IORING_OP_TIMEOUT_REMOVE] = {574 .name = "TIMEOUT_REMOVE",575 },576 [IORING_OP_ACCEPT] = {577 .name = "ACCEPT",578 },579 [IORING_OP_ASYNC_CANCEL] = {580 .name = "ASYNC_CANCEL",581 },582 [IORING_OP_LINK_TIMEOUT] = {583 .name = "LINK_TIMEOUT",584 },585 [IORING_OP_CONNECT] = {586 .name = "CONNECT",587 },588 [IORING_OP_FALLOCATE] = {589 .name = "FALLOCATE",590 },591 [IORING_OP_OPENAT] = {592 .name = "OPENAT",593 .cleanup = io_open_cleanup,594 },595 [IORING_OP_CLOSE] = {596 .name = "CLOSE",597 },598 [IORING_OP_FILES_UPDATE] = {599 .name = "FILES_UPDATE",600 },601 [IORING_OP_STATX] = {602 .name = "STATX",603 .cleanup = io_statx_cleanup,604 },605 [IORING_OP_READ] = {606 .name = "READ",607 .cleanup = io_readv_writev_cleanup,608 .fail = io_rw_fail,609 },610 [IORING_OP_WRITE] = {611 .name = "WRITE",612 .cleanup = io_readv_writev_cleanup,613 .fail = io_rw_fail,614 },615 [IORING_OP_FADVISE] = {616 .name = "FADVISE",617 },618 [IORING_OP_MADVISE] = {619 .name = "MADVISE",620 },621 [IORING_OP_SEND] = {622 .name = "SEND",623#if defined(CONFIG_NET)624 .cleanup = io_sendmsg_recvmsg_cleanup,625 .fail = io_sendrecv_fail,626#endif627 },628 [IORING_OP_RECV] = {629 .name = "RECV",630#if defined(CONFIG_NET)631 .cleanup = io_sendmsg_recvmsg_cleanup,632 .fail = io_sendrecv_fail,633#endif634 },635 [IORING_OP_OPENAT2] = {636 .name = "OPENAT2",637 .cleanup = io_open_cleanup,638 },639 [IORING_OP_EPOLL_CTL] = {640 .name = "EPOLL",641 },642 [IORING_OP_SPLICE] = {643 .name = "SPLICE",644 },645 [IORING_OP_PROVIDE_BUFFERS] = {646 .name = "PROVIDE_BUFFERS",647 },648 [IORING_OP_REMOVE_BUFFERS] = {649 .name = "REMOVE_BUFFERS",650 },651 [IORING_OP_TEE] = {652 .name = "TEE",653 },654 [IORING_OP_SHUTDOWN] = {655 .name = "SHUTDOWN",656 },657 [IORING_OP_RENAMEAT] = {658 .name = "RENAMEAT",659 .cleanup = io_renameat_cleanup,660 },661 [IORING_OP_UNLINKAT] = {662 .name = "UNLINKAT",663 .cleanup = io_unlinkat_cleanup,664 },665 [IORING_OP_MKDIRAT] = {666 .name = "MKDIRAT",667 .cleanup = io_mkdirat_cleanup,668 },669 [IORING_OP_SYMLINKAT] = {670 .name = "SYMLINKAT",671 .cleanup = io_link_cleanup,672 },673 [IORING_OP_LINKAT] = {674 .name = "LINKAT",675 .cleanup = io_link_cleanup,676 },677 [IORING_OP_MSG_RING] = {678 .name = "MSG_RING",679 .cleanup = io_msg_ring_cleanup,680 },681 [IORING_OP_FSETXATTR] = {682 .name = "FSETXATTR",683 .cleanup = io_xattr_cleanup,684 },685 [IORING_OP_SETXATTR] = {686 .name = "SETXATTR",687 .cleanup = io_xattr_cleanup,688 },689 [IORING_OP_FGETXATTR] = {690 .name = "FGETXATTR",691 .cleanup = io_xattr_cleanup,692 },693 [IORING_OP_GETXATTR] = {694 .name = "GETXATTR",695 .cleanup = io_xattr_cleanup,696 },697 [IORING_OP_SOCKET] = {698 .name = "SOCKET",699 },700 [IORING_OP_URING_CMD] = {701 .name = "URING_CMD",702 },703 [IORING_OP_SEND_ZC] = {704 .name = "SEND_ZC",705#if defined(CONFIG_NET)706 .cleanup = io_send_zc_cleanup,707 .fail = io_sendrecv_fail,708#endif709 },710 [IORING_OP_SENDMSG_ZC] = {711 .name = "SENDMSG_ZC",712#if defined(CONFIG_NET)713 .cleanup = io_send_zc_cleanup,714 .fail = io_sendrecv_fail,715#endif716 },717 [IORING_OP_READ_MULTISHOT] = {718 .name = "READ_MULTISHOT",719 .cleanup = io_readv_writev_cleanup,720 },721 [IORING_OP_WAITID] = {722 .name = "WAITID",723 },724 [IORING_OP_FUTEX_WAIT] = {725 .name = "FUTEX_WAIT",726 },727 [IORING_OP_FUTEX_WAKE] = {728 .name = "FUTEX_WAKE",729 },730 [IORING_OP_FUTEX_WAITV] = {731 .name = "FUTEX_WAITV",732 },733 [IORING_OP_FIXED_FD_INSTALL] = {734 .name = "FIXED_FD_INSTALL",735 },736 [IORING_OP_FTRUNCATE] = {737 .name = "FTRUNCATE",738 },739 [IORING_OP_BIND] = {740 .name = "BIND",741 },742 [IORING_OP_LISTEN] = {743 .name = "LISTEN",744 },745};746 747const char *io_uring_get_opcode(u8 opcode)748{749 if (opcode < IORING_OP_LAST)750 return io_cold_defs[opcode].name;751 return "INVALID";752}753 754bool io_uring_op_supported(u8 opcode)755{756 if (opcode < IORING_OP_LAST &&757 io_issue_defs[opcode].prep != io_eopnotsupp_prep)758 return true;759 return false;760}761 762void __init io_uring_optable_init(void)763{764 int i;765 766 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);767 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);768 769 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {770 BUG_ON(!io_issue_defs[i].prep);771 if (io_issue_defs[i].prep != io_eopnotsupp_prep)772 BUG_ON(!io_issue_defs[i].issue);773 WARN_ON_ONCE(!io_cold_defs[i].name);774 }775}776