331 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <linux/kernel.h>3#include <linux/errno.h>4#include <linux/file.h>5#include <linux/slab.h>6#include <linux/nospec.h>7#include <linux/io_uring.h>8 9#include <uapi/linux/io_uring.h>10 11#include "io_uring.h"12#include "rsrc.h"13#include "filetable.h"14#include "alloc_cache.h"15#include "msg_ring.h"16 17/* All valid masks for MSG_RING */18#define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \19 IORING_MSG_RING_FLAGS_PASS)20 21struct io_msg {22 struct file *file;23 struct file *src_file;24 struct callback_head tw;25 u64 user_data;26 u32 len;27 u32 cmd;28 u32 src_fd;29 union {30 u32 dst_fd;31 u32 cqe_flags;32 };33 u32 flags;34};35 36static HWJS_SUSPENDS void io_double_unlock_ctx(struct io_ring_ctx *octx)37{38 mutex_unlock(&octx->uring_lock);39}40 41static int io_double_lock_ctx(struct io_ring_ctx *octx,42 unsigned int issue_flags)43{44 /*45 * To ensure proper ordering between the two ctxs, we can only46 * attempt a trylock on the target. If that fails and we already have47 * the source ctx lock, punt to io-wq.48 */49 if (!(issue_flags & IO_URING_F_UNLOCKED)) {50 if (!mutex_trylock(&octx->uring_lock))51 return -EAGAIN;52 return 0;53 }54 mutex_lock(&octx->uring_lock);55 return 0;56}57 58void io_msg_ring_cleanup(struct io_kiocb *req)59{60 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);61 62 if (WARN_ON_ONCE(!msg->src_file))63 return;64 65 fput(msg->src_file);66 msg->src_file = NULL;67}68 69static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)70{71 return target_ctx->task_complete;72}73 74static HWJS_SUSPENDS void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts)75{76 struct io_ring_ctx *ctx = req->ctx;77 78 io_add_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags);79 if (spin_trylock(&ctx->msg_lock)) {80 if (io_alloc_cache_put(&ctx->msg_cache, req))81 req = NULL;82 spin_unlock(&ctx->msg_lock);83 }84 if (req)85 kmem_cache_free(req_cachep, req);86 percpu_ref_put(&ctx->refs);87}88 89static HWJS_SUSPENDS int io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req,90 int res, u32 cflags, u64 user_data)91{92 req->task = READ_ONCE(ctx->submitter_task);93 if (!req->task) {94 kmem_cache_free(req_cachep, req);95 return -EOWNERDEAD;96 }97 req->cqe.user_data = user_data;98 io_req_set_res(req, res, cflags);99 percpu_ref_get(&ctx->refs);100 req->ctx = ctx;101 req->io_task_work.func = io_msg_tw_complete;102 io_req_task_work_add_remote(req, ctx, IOU_F_TWQ_LAZY_WAKE);103 return 0;104}105 106static struct io_kiocb *io_msg_get_kiocb(struct io_ring_ctx *ctx)107{108 struct io_kiocb *req = NULL;109 110 if (spin_trylock(&ctx->msg_lock)) {111 req = io_alloc_cache_get(&ctx->msg_cache);112 spin_unlock(&ctx->msg_lock);113 if (req)114 return req;115 }116 return kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);117}118 119static HWJS_SUSPENDS int io_msg_data_remote(struct io_kiocb *req)120{121 struct io_ring_ctx *target_ctx = req->file->private_data;122 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);123 struct io_kiocb *target;124 u32 flags = 0;125 126 target = io_msg_get_kiocb(req->ctx);127 if (unlikely(!target))128 return -ENOMEM;129 130 if (msg->flags & IORING_MSG_RING_FLAGS_PASS)131 flags = msg->cqe_flags;132 133 return io_msg_remote_post(target_ctx, target, msg->len, flags,134 msg->user_data);135}136 137static HWJS_SUSPENDS int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)138{139 struct io_ring_ctx *target_ctx = req->file->private_data;140 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);141 u32 flags = 0;142 int ret;143 144 if (msg->src_fd || msg->flags & ~IORING_MSG_RING_FLAGS_PASS)145 return -EINVAL;146 if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS) && msg->dst_fd)147 return -EINVAL;148 if (target_ctx->flags & IORING_SETUP_R_DISABLED)149 return -EBADFD;150 151 if (io_msg_need_remote(target_ctx))152 return io_msg_data_remote(req);153 154 if (msg->flags & IORING_MSG_RING_FLAGS_PASS)155 flags = msg->cqe_flags;156 157 ret = -EOVERFLOW;158 if (target_ctx->flags & IORING_SETUP_IOPOLL) {159 if (unlikely(io_double_lock_ctx(target_ctx, issue_flags)))160 return -EAGAIN;161 }162 if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, flags))163 ret = 0;164 if (target_ctx->flags & IORING_SETUP_IOPOLL)165 io_double_unlock_ctx(target_ctx);166 return ret;167}168 169static HWJS_SUSPENDS struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)170{171 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);172 struct io_ring_ctx *ctx = req->ctx;173 struct file *file = NULL;174 int idx = msg->src_fd;175 176 io_ring_submit_lock(ctx, issue_flags);177 if (likely(idx < ctx->nr_user_files)) {178 idx = array_index_nospec(idx, ctx->nr_user_files);179 file = io_file_from_index(&ctx->file_table, idx);180 if (file)181 get_file(file);182 }183 io_ring_submit_unlock(ctx, issue_flags);184 return file;185}186 187static HWJS_SUSPENDS int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flags)188{189 struct io_ring_ctx *target_ctx = req->file->private_data;190 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);191 struct file *src_file = msg->src_file;192 int ret;193 194 if (unlikely(io_double_lock_ctx(target_ctx, issue_flags)))195 return -EAGAIN;196 197 ret = __io_fixed_fd_install(target_ctx, src_file, msg->dst_fd);198 if (ret < 0)199 goto out_unlock;200 201 msg->src_file = NULL;202 req->flags &= ~REQ_F_NEED_CLEANUP;203 204 if (msg->flags & IORING_MSG_RING_CQE_SKIP)205 goto out_unlock;206 /*207 * If this fails, the target still received the file descriptor but208 * wasn't notified of the fact. This means that if this request209 * completes with -EOVERFLOW, then the sender must ensure that a210 * later IORING_OP_MSG_RING delivers the message.211 */212 if (!io_post_aux_cqe(target_ctx, msg->user_data, ret, 0))213 ret = -EOVERFLOW;214out_unlock:215 io_double_unlock_ctx(target_ctx);216 return ret;217}218 219static HWJS_SUSPENDS void io_msg_tw_fd_complete(struct callback_head *head)220{221 struct io_msg *msg = container_of(head, struct io_msg, tw);222 struct io_kiocb *req = cmd_to_io_kiocb(msg);223 int ret = -EOWNERDEAD;224 225 if (!(current->flags & PF_EXITING))226 ret = io_msg_install_complete(req, IO_URING_F_UNLOCKED);227 if (ret < 0)228 req_set_fail(req);229 io_req_queue_tw_complete(req, ret);230}231 232static HWJS_SUSPENDS int io_msg_fd_remote(struct io_kiocb *req)233{234 struct io_ring_ctx *ctx = req->file->private_data;235 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);236 struct task_struct *task = READ_ONCE(ctx->submitter_task);237 238 if (unlikely(!task))239 return -EOWNERDEAD;240 241 init_task_work(&msg->tw, io_msg_tw_fd_complete);242 if (task_work_add(task, &msg->tw, TWA_SIGNAL))243 return -EOWNERDEAD;244 245 return IOU_ISSUE_SKIP_COMPLETE;246}247 248static HWJS_SUSPENDS int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)249{250 struct io_ring_ctx *target_ctx = req->file->private_data;251 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);252 struct io_ring_ctx *ctx = req->ctx;253 struct file *src_file = msg->src_file;254 255 if (msg->len)256 return -EINVAL;257 if (target_ctx == ctx)258 return -EINVAL;259 if (target_ctx->flags & IORING_SETUP_R_DISABLED)260 return -EBADFD;261 if (!src_file) {262 src_file = io_msg_grab_file(req, issue_flags);263 if (!src_file)264 return -EBADF;265 msg->src_file = src_file;266 req->flags |= REQ_F_NEED_CLEANUP;267 }268 269 if (io_msg_need_remote(target_ctx))270 return io_msg_fd_remote(req);271 return io_msg_install_complete(req, issue_flags);272}273 274int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)275{276 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);277 278 if (unlikely(sqe->buf_index || sqe->personality))279 return -EINVAL;280 281 msg->src_file = NULL;282 msg->user_data = READ_ONCE(sqe->off);283 msg->len = READ_ONCE(sqe->len);284 msg->cmd = READ_ONCE(sqe->addr);285 msg->src_fd = READ_ONCE(sqe->addr3);286 msg->dst_fd = READ_ONCE(sqe->file_index);287 msg->flags = READ_ONCE(sqe->msg_ring_flags);288 if (msg->flags & ~IORING_MSG_RING_MASK)289 return -EINVAL;290 291 return 0;292}293 294int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)295{296 struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);297 int ret;298 299 ret = -EBADFD;300 if (!io_is_uring_fops(req->file))301 goto done;302 303 switch (msg->cmd) {304 case IORING_MSG_DATA:305 ret = io_msg_ring_data(req, issue_flags);306 break;307 case IORING_MSG_SEND_FD:308 ret = io_msg_send_fd(req, issue_flags);309 break;310 default:311 ret = -EINVAL;312 break;313 }314 315done:316 if (ret < 0) {317 if (ret == -EAGAIN || ret == IOU_ISSUE_SKIP_COMPLETE)318 return ret;319 req_set_fail(req);320 }321 io_req_set_res(req, ret, 0);322 return IOU_OK;323}324 325void io_msg_cache_free(const void *entry)326{327 struct io_kiocb *req = (struct io_kiocb *) entry;328 329 kmem_cache_free(req_cachep, req);330}331