493 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#ifndef _LINUX_CLOSURE_H3#define _LINUX_CLOSURE_H4 5#include <linux/llist.h>6#include <linux/sched.h>7#include <linux/sched/task_stack.h>8#include <linux/workqueue.h>9 10/*11 * Closure is perhaps the most overused and abused term in computer science, but12 * since I've been unable to come up with anything better you're stuck with it13 * again.14 *15 * What are closures?16 *17 * They embed a refcount. The basic idea is they count "things that are in18 * progress" - in flight bios, some other thread that's doing something else -19 * anything you might want to wait on.20 *21 * The refcount may be manipulated with closure_get() and closure_put().22 * closure_put() is where many of the interesting things happen, when it causes23 * the refcount to go to 0.24 *25 * Closures can be used to wait on things both synchronously and asynchronously,26 * and synchronous and asynchronous use can be mixed without restriction. To27 * wait synchronously, use closure_sync() - you will sleep until your closure's28 * refcount hits 1.29 *30 * To wait asynchronously, use31 * continue_at(cl, next_function, workqueue);32 *33 * passing it, as you might expect, the function to run when nothing is pending34 * and the workqueue to run that function out of.35 *36 * continue_at() also, critically, requires a 'return' immediately following the37 * location where this macro is referenced, to return to the calling function.38 * There's good reason for this.39 *40 * To use safely closures asynchronously, they must always have a refcount while41 * they are running owned by the thread that is running them. Otherwise, suppose42 * you submit some bios and wish to have a function run when they all complete:43 *44 * foo_endio(struct bio *bio)45 * {46 * closure_put(cl);47 * }48 *49 * closure_init(cl);50 *51 * do_stuff();52 * closure_get(cl);53 * bio1->bi_endio = foo_endio;54 * bio_submit(bio1);55 *56 * do_more_stuff();57 * closure_get(cl);58 * bio2->bi_endio = foo_endio;59 * bio_submit(bio2);60 *61 * continue_at(cl, complete_some_read, system_wq);62 *63 * If closure's refcount started at 0, complete_some_read() could run before the64 * second bio was submitted - which is almost always not what you want! More65 * importantly, it wouldn't be possible to say whether the original thread or66 * complete_some_read()'s thread owned the closure - and whatever state it was67 * associated with!68 *69 * So, closure_init() initializes a closure's refcount to 1 - and when a70 * closure_fn is run, the refcount will be reset to 1 first.71 *72 * Then, the rule is - if you got the refcount with closure_get(), release it73 * with closure_put() (i.e, in a bio->bi_endio function). If you have a refcount74 * on a closure because you called closure_init() or you were run out of a75 * closure - _always_ use continue_at(). Doing so consistently will help76 * eliminate an entire class of particularly pernicious races.77 *78 * Lastly, you might have a wait list dedicated to a specific event, and have no79 * need for specifying the condition - you just want to wait until someone runs80 * closure_wake_up() on the appropriate wait list. In that case, just use81 * closure_wait(). It will return either true or false, depending on whether the82 * closure was already on a wait list or not - a closure can only be on one wait83 * list at a time.84 *85 * Parents:86 *87 * closure_init() takes two arguments - it takes the closure to initialize, and88 * a (possibly null) parent.89 *90 * If parent is non null, the new closure will have a refcount for its lifetime;91 * a closure is considered to be "finished" when its refcount hits 0 and the92 * function to run is null. Hence93 *94 * continue_at(cl, NULL, NULL);95 *96 * returns up the (spaghetti) stack of closures, precisely like normal return97 * returns up the C stack. continue_at() with non null fn is better thought of98 * as doing a tail call.99 *100 * All this implies that a closure should typically be embedded in a particular101 * struct (which its refcount will normally control the lifetime of), and that102 * struct can very much be thought of as a stack frame.103 */104 105struct closure;106struct closure_syncer;107typedef void (closure_fn) (struct work_struct *);108extern struct dentry *bcache_debug;109 110struct closure_waitlist {111 struct llist_head list;112};113 114enum closure_state {115 /*116 * CLOSURE_WAITING: Set iff the closure is on a waitlist. Must be set by117 * the thread that owns the closure, and cleared by the thread that's118 * waking up the closure.119 *120 * The rest are for debugging and don't affect behaviour:121 *122 * CLOSURE_RUNNING: Set when a closure is running (i.e. by123 * closure_init() and when closure_put() runs then next function), and124 * must be cleared before remaining hits 0. Primarily to help guard125 * against incorrect usage and accidentally transferring references.126 * continue_at() and closure_return() clear it for you, if you're doing127 * something unusual you can use closure_set_dead() which also helps128 * annotate where references are being transferred.129 */130 131 CLOSURE_BITS_START = (1U << 26),132 CLOSURE_DESTRUCTOR = (1U << 26),133 CLOSURE_WAITING = (1U << 28),134 CLOSURE_RUNNING = (1U << 30),135};136 137#define CLOSURE_GUARD_MASK \138 ((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_RUNNING) << 1)139 140#define CLOSURE_REMAINING_MASK (CLOSURE_BITS_START - 1)141#define CLOSURE_REMAINING_INITIALIZER (1|CLOSURE_RUNNING)142 143struct closure {144 union {145 struct {146 struct workqueue_struct *wq;147 struct closure_syncer *s;148 struct llist_node list;149 closure_fn *fn;150 };151 struct work_struct work;152 };153 154 struct closure *parent;155 156 atomic_t remaining;157 bool closure_get_happened;158 159#ifdef CONFIG_DEBUG_CLOSURES160#define CLOSURE_MAGIC_DEAD 0xc054dead161#define CLOSURE_MAGIC_ALIVE 0xc054a11e162#define CLOSURE_MAGIC_STACK 0xc05451cc163 164 unsigned int magic;165 struct list_head all;166 unsigned long ip;167 unsigned long waiting_on;168#endif169};170 171void closure_sub(struct closure *cl, int v);172void closure_put(struct closure *cl);173void __closure_wake_up(struct closure_waitlist *list);174bool closure_wait(struct closure_waitlist *list, struct closure *cl);175void __closure_sync(struct closure *cl);176 177static inline unsigned closure_nr_remaining(struct closure *cl)178{179 return atomic_read(&cl->remaining) & CLOSURE_REMAINING_MASK;180}181 182/**183 * closure_sync - sleep until a closure a closure has nothing left to wait on184 *185 * Sleeps until the refcount hits 1 - the thread that's running the closure owns186 * the last refcount.187 */188static inline void closure_sync(struct closure *cl)189{190#ifdef CONFIG_DEBUG_CLOSURES191 BUG_ON(closure_nr_remaining(cl) != 1 && !cl->closure_get_happened);192#endif193 194 if (cl->closure_get_happened)195 __closure_sync(cl);196}197 198int __closure_sync_timeout(struct closure *cl, unsigned long timeout);199 200static inline int closure_sync_timeout(struct closure *cl, unsigned long timeout)201{202#ifdef CONFIG_DEBUG_CLOSURES203 BUG_ON(closure_nr_remaining(cl) != 1 && !cl->closure_get_happened);204#endif205 return cl->closure_get_happened206 ? __closure_sync_timeout(cl, timeout)207 : 0;208}209 210#ifdef CONFIG_DEBUG_CLOSURES211 212void closure_debug_create(struct closure *cl);213void closure_debug_destroy(struct closure *cl);214 215#else216 217static inline void closure_debug_create(struct closure *cl) {}218static inline void closure_debug_destroy(struct closure *cl) {}219 220#endif221 222static inline void closure_set_ip(struct closure *cl)223{224#ifdef CONFIG_DEBUG_CLOSURES225 cl->ip = _THIS_IP_;226#endif227}228 229static inline void closure_set_ret_ip(struct closure *cl)230{231#ifdef CONFIG_DEBUG_CLOSURES232 cl->ip = _RET_IP_;233#endif234}235 236static inline void closure_set_waiting(struct closure *cl, unsigned long f)237{238#ifdef CONFIG_DEBUG_CLOSURES239 cl->waiting_on = f;240#endif241}242 243static inline void closure_set_stopped(struct closure *cl)244{245 atomic_sub(CLOSURE_RUNNING, &cl->remaining);246}247 248static inline void set_closure_fn(struct closure *cl, closure_fn *fn,249 struct workqueue_struct *wq)250{251 closure_set_ip(cl);252 cl->fn = fn;253 cl->wq = wq;254}255 256static inline void closure_queue(struct closure *cl)257{258 struct workqueue_struct *wq = cl->wq;259 /**260 * Changes made to closure, work_struct, or a couple of other structs261 * may cause work.func not pointing to the right location.262 */263 BUILD_BUG_ON(offsetof(struct closure, fn)264 != offsetof(struct work_struct, func));265 266 if (wq) {267 INIT_WORK(&cl->work, cl->work.func);268 BUG_ON(!queue_work(wq, &cl->work));269 } else270 cl->fn(&cl->work);271}272 273/**274 * closure_get - increment a closure's refcount275 */276static inline void closure_get(struct closure *cl)277{278 cl->closure_get_happened = true;279 280#ifdef CONFIG_DEBUG_CLOSURES281 BUG_ON((atomic_inc_return(&cl->remaining) &282 CLOSURE_REMAINING_MASK) <= 1);283#else284 atomic_inc(&cl->remaining);285#endif286}287 288/**289 * closure_get_not_zero290 */291static inline bool closure_get_not_zero(struct closure *cl)292{293 unsigned old = atomic_read(&cl->remaining);294 do {295 if (!(old & CLOSURE_REMAINING_MASK))296 return false;297 298 } while (!atomic_try_cmpxchg_acquire(&cl->remaining, &old, old + 1));299 300 return true;301}302 303/**304 * closure_init - Initialize a closure, setting the refcount to 1305 * @cl: closure to initialize306 * @parent: parent of the new closure. cl will take a refcount on it for its307 * lifetime; may be NULL.308 */309static inline void closure_init(struct closure *cl, struct closure *parent)310{311 cl->fn = NULL;312 cl->parent = parent;313 if (parent)314 closure_get(parent);315 316 atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);317 cl->closure_get_happened = false;318 319 closure_debug_create(cl);320 closure_set_ip(cl);321}322 323static inline void closure_init_stack(struct closure *cl)324{325 memset(cl, 0, sizeof(struct closure));326 atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);327#ifdef CONFIG_DEBUG_CLOSURES328 cl->magic = CLOSURE_MAGIC_STACK;329#endif330}331 332static inline void closure_init_stack_release(struct closure *cl)333{334 memset(cl, 0, sizeof(struct closure));335 atomic_set_release(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);336#ifdef CONFIG_DEBUG_CLOSURES337 cl->magic = CLOSURE_MAGIC_STACK;338#endif339}340 341/**342 * closure_wake_up - wake up all closures on a wait list,343 * with memory barrier344 */345static inline void closure_wake_up(struct closure_waitlist *list)346{347 /* Memory barrier for the wait list */348 smp_mb();349 __closure_wake_up(list);350}351 352#define CLOSURE_CALLBACK(name) void name(struct work_struct *ws)353#define closure_type(name, type, member) \354 struct closure *cl = container_of(ws, struct closure, work); \355 type *name = container_of(cl, type, member)356 357/**358 * continue_at - jump to another function with barrier359 *360 * After @cl is no longer waiting on anything (i.e. all outstanding refs have361 * been dropped with closure_put()), it will resume execution at @fn running out362 * of @wq (or, if @wq is NULL, @fn will be called by closure_put() directly).363 *364 * This is because after calling continue_at() you no longer have a ref on @cl,365 * and whatever @cl owns may be freed out from under you - a running closure fn366 * has a ref on its own closure which continue_at() drops.367 *368 * Note you are expected to immediately return after using this macro.369 */370#define continue_at(_cl, _fn, _wq) \371do { \372 set_closure_fn(_cl, _fn, _wq); \373 closure_sub(_cl, CLOSURE_RUNNING + 1); \374} while (0)375 376/**377 * closure_return - finish execution of a closure378 *379 * This is used to indicate that @cl is finished: when all outstanding refs on380 * @cl have been dropped @cl's ref on its parent closure (as passed to381 * closure_init()) will be dropped, if one was specified - thus this can be382 * thought of as returning to the parent closure.383 */384#define closure_return(_cl) continue_at((_cl), NULL, NULL)385 386void closure_return_sync(struct closure *cl);387 388/**389 * continue_at_nobarrier - jump to another function without barrier390 *391 * Causes @fn to be executed out of @cl, in @wq context (or called directly if392 * @wq is NULL).393 *394 * The ref the caller of continue_at_nobarrier() had on @cl is now owned by @fn,395 * thus it's not safe to touch anything protected by @cl after a396 * continue_at_nobarrier().397 */398#define continue_at_nobarrier(_cl, _fn, _wq) \399do { \400 set_closure_fn(_cl, _fn, _wq); \401 closure_queue(_cl); \402} while (0)403 404/**405 * closure_return_with_destructor - finish execution of a closure,406 * with destructor407 *408 * Works like closure_return(), except @destructor will be called when all409 * outstanding refs on @cl have been dropped; @destructor may be used to safely410 * free the memory occupied by @cl, and it is called with the ref on the parent411 * closure still held - so @destructor could safely return an item to a412 * freelist protected by @cl's parent.413 */414#define closure_return_with_destructor(_cl, _destructor) \415do { \416 set_closure_fn(_cl, _destructor, NULL); \417 closure_sub(_cl, CLOSURE_RUNNING - CLOSURE_DESTRUCTOR + 1); \418} while (0)419 420/**421 * closure_call - execute @fn out of a new, uninitialized closure422 *423 * Typically used when running out of one closure, and we want to run @fn424 * asynchronously out of a new closure - @parent will then wait for @cl to425 * finish.426 */427static inline void closure_call(struct closure *cl, closure_fn fn,428 struct workqueue_struct *wq,429 struct closure *parent)430{431 closure_init(cl, parent);432 continue_at_nobarrier(cl, fn, wq);433}434 435#define __closure_wait_event(waitlist, _cond) \436do { \437 struct closure cl; \438 \439 closure_init_stack(&cl); \440 \441 while (1) { \442 closure_wait(waitlist, &cl); \443 if (_cond) \444 break; \445 closure_sync(&cl); \446 } \447 closure_wake_up(waitlist); \448 closure_sync(&cl); \449} while (0)450 451#define closure_wait_event(waitlist, _cond) \452do { \453 if (!(_cond)) \454 __closure_wait_event(waitlist, _cond); \455} while (0)456 457#define __closure_wait_event_timeout(waitlist, _cond, _until) \458({ \459 struct closure cl; \460 long _t; \461 \462 closure_init_stack(&cl); \463 \464 while (1) { \465 closure_wait(waitlist, &cl); \466 if (_cond) { \467 _t = max_t(long, 1L, _until - jiffies); \468 break; \469 } \470 _t = max_t(long, 0L, _until - jiffies); \471 if (!_t) \472 break; \473 closure_sync_timeout(&cl, _t); \474 } \475 closure_wake_up(waitlist); \476 closure_sync(&cl); \477 _t; \478})479 480/*481 * Returns 0 if timeout expired, remaining time in jiffies (at least 1) if482 * condition became true483 */484#define closure_wait_event_timeout(waitlist, _cond, _timeout) \485({ \486 unsigned long _until = jiffies + _timeout; \487 (_cond) \488 ? max_t(long, 1L, _until - jiffies) \489 : __closure_wait_event_timeout(waitlist, _cond, _until);\490})491 492#endif /* _LINUX_CLOSURE_H */493