746 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2#include <linux/anon_inodes.h>3#include <linux/uio.h>4#include "internal.h"5 6struct ondemand_anon_file {7 struct file *file;8 int fd;9};10 11static inline void cachefiles_req_put(struct cachefiles_req *req)12{13 if (refcount_dec_and_test(&req->ref))14 kfree(req);15}16 17static int cachefiles_ondemand_fd_release(struct inode *inode,18 struct file *file)19{20 struct cachefiles_object *object = file->private_data;21 struct cachefiles_cache *cache;22 struct cachefiles_ondemand_info *info;23 int object_id;24 struct cachefiles_req *req;25 XA_STATE(xas, NULL, 0);26 27 if (!object)28 return 0;29 30 info = object->ondemand;31 cache = object->volume->cache;32 xas.xa = &cache->reqs;33 34 xa_lock(&cache->reqs);35 spin_lock(&info->lock);36 object_id = info->ondemand_id;37 info->ondemand_id = CACHEFILES_ONDEMAND_ID_CLOSED;38 cachefiles_ondemand_set_object_close(object);39 spin_unlock(&info->lock);40 41 /* Only flush CACHEFILES_REQ_NEW marked req to avoid race with daemon_read */42 xas_for_each_marked(&xas, req, ULONG_MAX, CACHEFILES_REQ_NEW) {43 if (req->msg.object_id == object_id &&44 req->msg.opcode == CACHEFILES_OP_CLOSE) {45 complete(&req->done);46 xas_store(&xas, NULL);47 }48 }49 xa_unlock(&cache->reqs);50 51 xa_erase(&cache->ondemand_ids, object_id);52 trace_cachefiles_ondemand_fd_release(object, object_id);53 cachefiles_put_object(object, cachefiles_obj_put_ondemand_fd);54 cachefiles_put_unbind_pincount(cache);55 return 0;56}57 58static ssize_t cachefiles_ondemand_fd_write_iter(struct kiocb *kiocb,59 struct iov_iter *iter)60{61 struct cachefiles_object *object = kiocb->ki_filp->private_data;62 struct cachefiles_cache *cache = object->volume->cache;63 struct file *file = object->file;64 size_t len = iter->count;65 loff_t pos = kiocb->ki_pos;66 const struct cred *saved_cred;67 int ret;68 69 if (!file)70 return -ENOBUFS;71 72 cachefiles_begin_secure(cache, &saved_cred);73 ret = __cachefiles_prepare_write(object, file, &pos, &len, len, true);74 cachefiles_end_secure(cache, saved_cred);75 if (ret < 0)76 return ret;77 78 trace_cachefiles_ondemand_fd_write(object, file_inode(file), pos, len);79 ret = __cachefiles_write(object, file, pos, iter, NULL, NULL);80 if (!ret)81 ret = len;82 83 return ret;84}85 86static loff_t cachefiles_ondemand_fd_llseek(struct file *filp, loff_t pos,87 int whence)88{89 struct cachefiles_object *object = filp->private_data;90 struct file *file = object->file;91 92 if (!file)93 return -ENOBUFS;94 95 return vfs_llseek(file, pos, whence);96}97 98static long cachefiles_ondemand_fd_ioctl(struct file *filp, unsigned int ioctl,99 unsigned long id)100{101 struct cachefiles_object *object = filp->private_data;102 struct cachefiles_cache *cache = object->volume->cache;103 struct cachefiles_req *req;104 XA_STATE(xas, &cache->reqs, id);105 106 if (ioctl != CACHEFILES_IOC_READ_COMPLETE)107 return -EINVAL;108 109 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags))110 return -EOPNOTSUPP;111 112 xa_lock(&cache->reqs);113 req = xas_load(&xas);114 if (!req || req->msg.opcode != CACHEFILES_OP_READ ||115 req->object != object) {116 xa_unlock(&cache->reqs);117 return -EINVAL;118 }119 xas_store(&xas, NULL);120 xa_unlock(&cache->reqs);121 122 trace_cachefiles_ondemand_cread(object, id);123 complete(&req->done);124 return 0;125}126 127static const struct file_operations cachefiles_ondemand_fd_fops = {128 .owner = THIS_MODULE,129 .release = cachefiles_ondemand_fd_release,130 .write_iter = cachefiles_ondemand_fd_write_iter,131 .llseek = cachefiles_ondemand_fd_llseek,132 .unlocked_ioctl = cachefiles_ondemand_fd_ioctl,133};134 135/*136 * OPEN request Completion (copen)137 * - command: "copen <id>,<cache_size>"138 * <cache_size> indicates the object size if >=0, error code if negative139 */140int cachefiles_ondemand_copen(struct cachefiles_cache *cache, char *args)141{142 struct cachefiles_req *req;143 struct fscache_cookie *cookie;144 struct cachefiles_ondemand_info *info;145 char *pid, *psize;146 unsigned long id;147 long size;148 int ret;149 XA_STATE(xas, &cache->reqs, 0);150 151 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags))152 return -EOPNOTSUPP;153 154 if (!*args) {155 pr_err("Empty id specified\n");156 return -EINVAL;157 }158 159 pid = args;160 psize = strchr(args, ',');161 if (!psize) {162 pr_err("Cache size is not specified\n");163 return -EINVAL;164 }165 166 *psize = 0;167 psize++;168 169 ret = kstrtoul(pid, 0, &id);170 if (ret)171 return ret;172 173 xa_lock(&cache->reqs);174 xas.xa_index = id;175 req = xas_load(&xas);176 if (!req || req->msg.opcode != CACHEFILES_OP_OPEN ||177 !req->object->ondemand->ondemand_id) {178 xa_unlock(&cache->reqs);179 return -EINVAL;180 }181 xas_store(&xas, NULL);182 xa_unlock(&cache->reqs);183 184 info = req->object->ondemand;185 /* fail OPEN request if copen format is invalid */186 ret = kstrtol(psize, 0, &size);187 if (ret) {188 req->error = ret;189 goto out;190 }191 192 /* fail OPEN request if daemon reports an error */193 if (size < 0) {194 if (!IS_ERR_VALUE(size)) {195 req->error = -EINVAL;196 ret = -EINVAL;197 } else {198 req->error = size;199 ret = 0;200 }201 goto out;202 }203 204 spin_lock(&info->lock);205 /*206 * The anonymous fd was closed before copen ? Fail the request.207 *208 * t1 | t2209 * ---------------------------------------------------------210 * cachefiles_ondemand_copen211 * req = xa_erase(&cache->reqs, id)212 * // Anon fd is maliciously closed.213 * cachefiles_ondemand_fd_release214 * xa_lock(&cache->reqs)215 * cachefiles_ondemand_set_object_close(object)216 * xa_unlock(&cache->reqs)217 * cachefiles_ondemand_set_object_open218 * // No one will ever close it again.219 * cachefiles_ondemand_daemon_read220 * cachefiles_ondemand_select_req221 *222 * Get a read req but its fd is already closed. The daemon can't223 * issue a cread ioctl with an closed fd, then hung.224 */225 if (info->ondemand_id == CACHEFILES_ONDEMAND_ID_CLOSED) {226 spin_unlock(&info->lock);227 req->error = -EBADFD;228 goto out;229 }230 cookie = req->object->cookie;231 cookie->object_size = size;232 if (size)233 clear_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);234 else235 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);236 trace_cachefiles_ondemand_copen(req->object, id, size);237 238 cachefiles_ondemand_set_object_open(req->object);239 spin_unlock(&info->lock);240 wake_up_all(&cache->daemon_pollwq);241 242out:243 spin_lock(&info->lock);244 /* Need to set object close to avoid reopen status continuing */245 if (info->ondemand_id == CACHEFILES_ONDEMAND_ID_CLOSED)246 cachefiles_ondemand_set_object_close(req->object);247 spin_unlock(&info->lock);248 complete(&req->done);249 return ret;250}251 252int cachefiles_ondemand_restore(struct cachefiles_cache *cache, char *args)253{254 struct cachefiles_req *req;255 256 XA_STATE(xas, &cache->reqs, 0);257 258 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags))259 return -EOPNOTSUPP;260 261 /*262 * Reset the requests to CACHEFILES_REQ_NEW state, so that the263 * requests have been processed halfway before the crash of the264 * user daemon could be reprocessed after the recovery.265 */266 xas_lock(&xas);267 xas_for_each(&xas, req, ULONG_MAX)268 xas_set_mark(&xas, CACHEFILES_REQ_NEW);269 xas_unlock(&xas);270 271 wake_up_all(&cache->daemon_pollwq);272 return 0;273}274 275static int cachefiles_ondemand_get_fd(struct cachefiles_req *req,276 struct ondemand_anon_file *anon_file)277{278 struct cachefiles_object *object;279 struct cachefiles_cache *cache;280 struct cachefiles_open *load;281 u32 object_id;282 int ret;283 284 object = cachefiles_grab_object(req->object,285 cachefiles_obj_get_ondemand_fd);286 cache = object->volume->cache;287 288 ret = xa_alloc_cyclic(&cache->ondemand_ids, &object_id, NULL,289 XA_LIMIT(1, INT_MAX),290 &cache->ondemand_id_next, GFP_KERNEL);291 if (ret < 0)292 goto err;293 294 anon_file->fd = get_unused_fd_flags(O_WRONLY);295 if (anon_file->fd < 0) {296 ret = anon_file->fd;297 goto err_free_id;298 }299 300 anon_file->file = anon_inode_getfile("[cachefiles]",301 &cachefiles_ondemand_fd_fops, object, O_WRONLY);302 if (IS_ERR(anon_file->file)) {303 ret = PTR_ERR(anon_file->file);304 goto err_put_fd;305 }306 307 spin_lock(&object->ondemand->lock);308 if (object->ondemand->ondemand_id > 0) {309 spin_unlock(&object->ondemand->lock);310 /* Pair with check in cachefiles_ondemand_fd_release(). */311 anon_file->file->private_data = NULL;312 ret = -EEXIST;313 goto err_put_file;314 }315 316 anon_file->file->f_mode |= FMODE_PWRITE | FMODE_LSEEK;317 318 load = (void *)req->msg.data;319 load->fd = anon_file->fd;320 object->ondemand->ondemand_id = object_id;321 spin_unlock(&object->ondemand->lock);322 323 cachefiles_get_unbind_pincount(cache);324 trace_cachefiles_ondemand_open(object, &req->msg, load);325 return 0;326 327err_put_file:328 fput(anon_file->file);329 anon_file->file = NULL;330err_put_fd:331 put_unused_fd(anon_file->fd);332 anon_file->fd = ret;333err_free_id:334 xa_erase(&cache->ondemand_ids, object_id);335err:336 spin_lock(&object->ondemand->lock);337 /* Avoid marking an opened object as closed. */338 if (object->ondemand->ondemand_id <= 0)339 cachefiles_ondemand_set_object_close(object);340 spin_unlock(&object->ondemand->lock);341 cachefiles_put_object(object, cachefiles_obj_put_ondemand_fd);342 return ret;343}344 345static void ondemand_object_worker(struct work_struct *work)346{347 struct cachefiles_ondemand_info *info =348 container_of(work, struct cachefiles_ondemand_info, ondemand_work);349 350 cachefiles_ondemand_init_object(info->object);351}352 353/*354 * If there are any inflight or subsequent READ requests on the355 * closed object, reopen it.356 * Skip read requests whose related object is reopening.357 */358static struct cachefiles_req *cachefiles_ondemand_select_req(struct xa_state *xas,359 unsigned long xa_max)360{361 struct cachefiles_req *req;362 struct cachefiles_object *object;363 struct cachefiles_ondemand_info *info;364 365 xas_for_each_marked(xas, req, xa_max, CACHEFILES_REQ_NEW) {366 if (req->msg.opcode != CACHEFILES_OP_READ)367 return req;368 object = req->object;369 info = object->ondemand;370 if (cachefiles_ondemand_object_is_close(object)) {371 cachefiles_ondemand_set_object_reopening(object);372 queue_work(fscache_wq, &info->ondemand_work);373 continue;374 }375 if (cachefiles_ondemand_object_is_reopening(object))376 continue;377 return req;378 }379 return NULL;380}381 382static inline bool cachefiles_ondemand_finish_req(struct cachefiles_req *req,383 struct xa_state *xas, int err)384{385 if (unlikely(!xas || !req))386 return false;387 388 if (xa_cmpxchg(xas->xa, xas->xa_index, req, NULL, 0) != req)389 return false;390 391 req->error = err;392 complete(&req->done);393 return true;394}395 396ssize_t cachefiles_ondemand_daemon_read(struct cachefiles_cache *cache,397 char __user *_buffer, size_t buflen)398{399 struct cachefiles_req *req;400 struct cachefiles_msg *msg;401 size_t n;402 int ret = 0;403 struct ondemand_anon_file anon_file;404 XA_STATE(xas, &cache->reqs, cache->req_id_next);405 406 xa_lock(&cache->reqs);407 /*408 * Cyclically search for a request that has not ever been processed,409 * to prevent requests from being processed repeatedly, and make410 * request distribution fair.411 */412 req = cachefiles_ondemand_select_req(&xas, ULONG_MAX);413 if (!req && cache->req_id_next > 0) {414 xas_set(&xas, 0);415 req = cachefiles_ondemand_select_req(&xas, cache->req_id_next - 1);416 }417 if (!req) {418 xa_unlock(&cache->reqs);419 return 0;420 }421 422 msg = &req->msg;423 n = msg->len;424 425 if (n > buflen) {426 xa_unlock(&cache->reqs);427 return -EMSGSIZE;428 }429 430 xas_clear_mark(&xas, CACHEFILES_REQ_NEW);431 cache->req_id_next = xas.xa_index + 1;432 refcount_inc(&req->ref);433 cachefiles_grab_object(req->object, cachefiles_obj_get_read_req);434 xa_unlock(&cache->reqs);435 436 if (msg->opcode == CACHEFILES_OP_OPEN) {437 ret = cachefiles_ondemand_get_fd(req, &anon_file);438 if (ret)439 goto out;440 }441 442 msg->msg_id = xas.xa_index;443 msg->object_id = req->object->ondemand->ondemand_id;444 445 if (copy_to_user(_buffer, msg, n) != 0)446 ret = -EFAULT;447 448 if (msg->opcode == CACHEFILES_OP_OPEN) {449 if (ret < 0) {450 fput(anon_file.file);451 put_unused_fd(anon_file.fd);452 goto out;453 }454 fd_install(anon_file.fd, anon_file.file);455 }456out:457 cachefiles_put_object(req->object, cachefiles_obj_put_read_req);458 /* Remove error request and CLOSE request has no reply */459 if (ret || msg->opcode == CACHEFILES_OP_CLOSE)460 cachefiles_ondemand_finish_req(req, &xas, ret);461 cachefiles_req_put(req);462 return ret ? ret : n;463}464 465typedef int (*init_req_fn)(struct cachefiles_req *req, void *private);466 467static int cachefiles_ondemand_send_req(struct cachefiles_object *object,468 enum cachefiles_opcode opcode,469 size_t data_len,470 init_req_fn init_req,471 void *private)472{473 struct cachefiles_cache *cache = object->volume->cache;474 struct cachefiles_req *req = NULL;475 XA_STATE(xas, &cache->reqs, 0);476 int ret;477 478 if (!test_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags))479 return 0;480 481 if (test_bit(CACHEFILES_DEAD, &cache->flags)) {482 ret = -EIO;483 goto out;484 }485 486 req = kzalloc(sizeof(*req) + data_len, GFP_KERNEL);487 if (!req) {488 ret = -ENOMEM;489 goto out;490 }491 492 refcount_set(&req->ref, 1);493 req->object = object;494 init_completion(&req->done);495 req->msg.opcode = opcode;496 req->msg.len = sizeof(struct cachefiles_msg) + data_len;497 498 ret = init_req(req, private);499 if (ret)500 goto out;501 502 do {503 /*504 * Stop enqueuing the request when daemon is dying. The505 * following two operations need to be atomic as a whole.506 * 1) check cache state, and507 * 2) enqueue request if cache is alive.508 * Otherwise the request may be enqueued after xarray has been509 * flushed, leaving the orphan request never being completed.510 *511 * CPU 1 CPU 2512 * ===== =====513 * test CACHEFILES_DEAD bit514 * set CACHEFILES_DEAD bit515 * flush requests in the xarray516 * enqueue the request517 */518 xas_lock(&xas);519 520 if (test_bit(CACHEFILES_DEAD, &cache->flags) ||521 cachefiles_ondemand_object_is_dropping(object)) {522 xas_unlock(&xas);523 ret = -EIO;524 goto out;525 }526 527 /* coupled with the barrier in cachefiles_flush_reqs() */528 smp_mb();529 530 if (opcode == CACHEFILES_OP_CLOSE &&531 !cachefiles_ondemand_object_is_open(object)) {532 WARN_ON_ONCE(object->ondemand->ondemand_id == 0);533 xas_unlock(&xas);534 ret = -EIO;535 goto out;536 }537 538 /*539 * Cyclically find a free xas to avoid msg_id reuse that would540 * cause the daemon to successfully copen a stale msg_id.541 */542 xas.xa_index = cache->msg_id_next;543 xas_find_marked(&xas, UINT_MAX, XA_FREE_MARK);544 if (xas.xa_node == XAS_RESTART) {545 xas.xa_index = 0;546 xas_find_marked(&xas, cache->msg_id_next - 1, XA_FREE_MARK);547 }548 if (xas.xa_node == XAS_RESTART)549 xas_set_err(&xas, -EBUSY);550 551 xas_store(&xas, req);552 if (xas_valid(&xas)) {553 cache->msg_id_next = xas.xa_index + 1;554 xas_clear_mark(&xas, XA_FREE_MARK);555 xas_set_mark(&xas, CACHEFILES_REQ_NEW);556 }557 xas_unlock(&xas);558 } while (xas_nomem(&xas, GFP_KERNEL));559 560 ret = xas_error(&xas);561 if (ret)562 goto out;563 564 wake_up_all(&cache->daemon_pollwq);565wait:566 ret = wait_for_completion_killable(&req->done);567 if (!ret) {568 ret = req->error;569 } else {570 ret = -EINTR;571 if (!cachefiles_ondemand_finish_req(req, &xas, ret)) {572 /* Someone will complete it soon. */573 cpu_relax();574 goto wait;575 }576 }577 cachefiles_req_put(req);578 return ret;579out:580 /* Reset the object to close state in error handling path.581 * If error occurs after creating the anonymous fd,582 * cachefiles_ondemand_fd_release() will set object to close.583 */584 if (opcode == CACHEFILES_OP_OPEN &&585 !cachefiles_ondemand_object_is_dropping(object))586 cachefiles_ondemand_set_object_close(object);587 kfree(req);588 return ret;589}590 591static int cachefiles_ondemand_init_open_req(struct cachefiles_req *req,592 void *private)593{594 struct cachefiles_object *object = req->object;595 struct fscache_cookie *cookie = object->cookie;596 struct fscache_volume *volume = object->volume->vcookie;597 struct cachefiles_open *load = (void *)req->msg.data;598 size_t volume_key_size, cookie_key_size;599 void *volume_key, *cookie_key;600 601 /*602 * Volume key is a NUL-terminated string. key[0] stores strlen() of the603 * string, followed by the content of the string (excluding '\0').604 */605 volume_key_size = volume->key[0] + 1;606 volume_key = volume->key + 1;607 608 /* Cookie key is binary data, which is netfs specific. */609 cookie_key_size = cookie->key_len;610 cookie_key = fscache_get_key(cookie);611 612 if (!(object->cookie->advice & FSCACHE_ADV_WANT_CACHE_SIZE)) {613 pr_err("WANT_CACHE_SIZE is needed for on-demand mode\n");614 return -EINVAL;615 }616 617 load->volume_key_size = volume_key_size;618 load->cookie_key_size = cookie_key_size;619 memcpy(load->data, volume_key, volume_key_size);620 memcpy(load->data + volume_key_size, cookie_key, cookie_key_size);621 622 return 0;623}624 625static int cachefiles_ondemand_init_close_req(struct cachefiles_req *req,626 void *private)627{628 struct cachefiles_object *object = req->object;629 630 if (!cachefiles_ondemand_object_is_open(object))631 return -ENOENT;632 633 trace_cachefiles_ondemand_close(object, &req->msg);634 return 0;635}636 637struct cachefiles_read_ctx {638 loff_t off;639 size_t len;640};641 642static int cachefiles_ondemand_init_read_req(struct cachefiles_req *req,643 void *private)644{645 struct cachefiles_object *object = req->object;646 struct cachefiles_read *load = (void *)req->msg.data;647 struct cachefiles_read_ctx *read_ctx = private;648 649 load->off = read_ctx->off;650 load->len = read_ctx->len;651 trace_cachefiles_ondemand_read(object, &req->msg, load);652 return 0;653}654 655int cachefiles_ondemand_init_object(struct cachefiles_object *object)656{657 struct fscache_cookie *cookie = object->cookie;658 struct fscache_volume *volume = object->volume->vcookie;659 size_t volume_key_size, cookie_key_size, data_len;660 661 if (!object->ondemand)662 return 0;663 664 /*665 * CacheFiles will firstly check the cache file under the root cache666 * directory. If the coherency check failed, it will fallback to667 * creating a new tmpfile as the cache file. Reuse the previously668 * allocated object ID if any.669 */670 if (cachefiles_ondemand_object_is_open(object))671 return 0;672 673 volume_key_size = volume->key[0] + 1;674 cookie_key_size = cookie->key_len;675 data_len = sizeof(struct cachefiles_open) +676 volume_key_size + cookie_key_size;677 678 return cachefiles_ondemand_send_req(object, CACHEFILES_OP_OPEN,679 data_len, cachefiles_ondemand_init_open_req, NULL);680}681 682void cachefiles_ondemand_clean_object(struct cachefiles_object *object)683{684 unsigned long index;685 struct cachefiles_req *req;686 struct cachefiles_cache *cache;687 688 if (!object->ondemand)689 return;690 691 cachefiles_ondemand_send_req(object, CACHEFILES_OP_CLOSE, 0,692 cachefiles_ondemand_init_close_req, NULL);693 694 if (!object->ondemand->ondemand_id)695 return;696 697 /* Cancel all requests for the object that is being dropped. */698 cache = object->volume->cache;699 xa_lock(&cache->reqs);700 cachefiles_ondemand_set_object_dropping(object);701 xa_for_each(&cache->reqs, index, req) {702 if (req->object == object) {703 req->error = -EIO;704 complete(&req->done);705 __xa_erase(&cache->reqs, index);706 }707 }708 xa_unlock(&cache->reqs);709 710 /* Wait for ondemand_object_worker() to finish to avoid UAF. */711 cancel_work_sync(&object->ondemand->ondemand_work);712}713 714int cachefiles_ondemand_init_obj_info(struct cachefiles_object *object,715 struct cachefiles_volume *volume)716{717 if (!cachefiles_in_ondemand_mode(volume->cache))718 return 0;719 720 object->ondemand = kzalloc(sizeof(struct cachefiles_ondemand_info),721 GFP_KERNEL);722 if (!object->ondemand)723 return -ENOMEM;724 725 object->ondemand->object = object;726 spin_lock_init(&object->ondemand->lock);727 INIT_WORK(&object->ondemand->ondemand_work, ondemand_object_worker);728 return 0;729}730 731void cachefiles_ondemand_deinit_obj_info(struct cachefiles_object *object)732{733 kfree(object->ondemand);734 object->ondemand = NULL;735}736 737int cachefiles_ondemand_read(struct cachefiles_object *object,738 loff_t pos, size_t len)739{740 struct cachefiles_read_ctx read_ctx = {pos, len};741 742 return cachefiles_ondemand_send_req(object, CACHEFILES_OP_READ,743 sizeof(struct cachefiles_read),744 cachefiles_ondemand_init_read_req, &read_ctx);745}746