176 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <linux/kernel.h>3#include <linux/errno.h>4#include <linux/file.h>5#include <linux/mm.h>6#include <linux/slab.h>7#include <linux/nospec.h>8#include <linux/io_uring.h>9 10#include <uapi/linux/io_uring.h>11 12#include "io_uring.h"13#include "rsrc.h"14#include "filetable.h"15 16static int io_file_bitmap_get(struct io_ring_ctx *ctx)17{18 struct io_file_table *table = &ctx->file_table;19 unsigned long nr = ctx->file_alloc_end;20 int ret;21 22 if (!table->bitmap)23 return -ENFILE;24 25 do {26 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);27 if (ret != nr)28 return ret;29 30 if (table->alloc_hint == ctx->file_alloc_start)31 break;32 nr = table->alloc_hint;33 table->alloc_hint = ctx->file_alloc_start;34 } while (1);35 36 return -ENFILE;37}38 39bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)40{41 table->files = kvcalloc(nr_files, sizeof(table->files[0]),42 GFP_KERNEL_ACCOUNT);43 if (unlikely(!table->files))44 return false;45 46 table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);47 if (unlikely(!table->bitmap)) {48 kvfree(table->files);49 return false;50 }51 52 return true;53}54 55void io_free_file_tables(struct io_file_table *table)56{57 kvfree(table->files);58 bitmap_free(table->bitmap);59 table->files = NULL;60 table->bitmap = NULL;61}62 63static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,64 u32 slot_index)65 __must_hold(&req->ctx->uring_lock)66{67 struct io_fixed_file *file_slot;68 int ret;69 70 if (io_is_uring_fops(file))71 return -EBADF;72 if (!ctx->file_data)73 return -ENXIO;74 if (slot_index >= ctx->nr_user_files)75 return -EINVAL;76 77 slot_index = array_index_nospec(slot_index, ctx->nr_user_files);78 file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);79 80 if (file_slot->file_ptr) {81 ret = io_queue_rsrc_removal(ctx->file_data, slot_index,82 io_slot_file(file_slot));83 if (ret)84 return ret;85 86 file_slot->file_ptr = 0;87 } else {88 io_file_bitmap_set(&ctx->file_table, slot_index);89 }90 91 *io_get_tag_slot(ctx->file_data, slot_index) = 0;92 io_fixed_file_set(file_slot, file);93 return 0;94}95 96int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file,97 unsigned int file_slot)98{99 bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;100 int ret;101 102 if (alloc_slot) {103 ret = io_file_bitmap_get(ctx);104 if (unlikely(ret < 0))105 return ret;106 file_slot = ret;107 } else {108 file_slot--;109 }110 111 ret = io_install_fixed_file(ctx, file, file_slot);112 if (!ret && alloc_slot)113 ret = file_slot;114 return ret;115}116/*117 * Note when io_fixed_fd_install() returns error value, it will ensure118 * fput() is called correspondingly.119 */120int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,121 struct file *file, unsigned int file_slot)122{123 struct io_ring_ctx *ctx = req->ctx;124 int ret;125 126 io_ring_submit_lock(ctx, issue_flags);127 ret = __io_fixed_fd_install(ctx, file, file_slot);128 io_ring_submit_unlock(ctx, issue_flags);129 130 if (unlikely(ret < 0))131 fput(file);132 return ret;133}134 135int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)136{137 struct io_fixed_file *file_slot;138 int ret;139 140 if (unlikely(!ctx->file_data))141 return -ENXIO;142 if (offset >= ctx->nr_user_files)143 return -EINVAL;144 145 offset = array_index_nospec(offset, ctx->nr_user_files);146 file_slot = io_fixed_file_slot(&ctx->file_table, offset);147 if (!file_slot->file_ptr)148 return -EBADF;149 150 ret = io_queue_rsrc_removal(ctx->file_data, offset,151 io_slot_file(file_slot));152 if (ret)153 return ret;154 155 file_slot->file_ptr = 0;156 io_file_bitmap_clear(&ctx->file_table, offset);157 return 0;158}159 160int io_register_file_alloc_range(struct io_ring_ctx *ctx,161 struct io_uring_file_index_range __user *arg)162{163 struct io_uring_file_index_range range;164 u32 end;165 166 if (copy_from_user(&range, arg, sizeof(range)))167 return -EFAULT;168 if (check_add_overflow(range.off, range.len, &end))169 return -EOVERFLOW;170 if (range.resv || end > ctx->nr_user_files)171 return -EINVAL;172 173 io_file_table_set_alloc_range(ctx, range.off, range.len);174 return 0;175}176