206 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2023 Red Hat4 */5 6#include "wait-queue.h"7 8#include <linux/device-mapper.h>9 10#include "permassert.h"11 12#include "status-codes.h"13 14/**15 * vdo_waitq_enqueue_waiter() - Add a waiter to the tail end of a waitq.16 * @waitq: The vdo_wait_queue to which to add the waiter.17 * @waiter: The waiter to add to the waitq.18 *19 * The waiter must not already be waiting in a waitq.20 */21void vdo_waitq_enqueue_waiter(struct vdo_wait_queue *waitq, struct vdo_waiter *waiter)22{23 BUG_ON(waiter->next_waiter != NULL);24 25 if (waitq->last_waiter == NULL) {26 /*27 * The waitq is empty, so form the initial circular list by self-linking the28 * initial waiter.29 */30 waiter->next_waiter = waiter;31 } else {32 /* Splice the new waiter in at the end of the waitq. */33 waiter->next_waiter = waitq->last_waiter->next_waiter;34 waitq->last_waiter->next_waiter = waiter;35 }36 37 /* In both cases, the waiter we added to the ring becomes the last waiter. */38 waitq->last_waiter = waiter;39 waitq->length += 1;40}41 42/**43 * vdo_waitq_transfer_all_waiters() - Transfer all waiters from one waitq to44 * a second waitq, emptying the first waitq.45 * @from_waitq: The waitq containing the waiters to move.46 * @to_waitq: The waitq that will receive the waiters from the first waitq.47 */48void vdo_waitq_transfer_all_waiters(struct vdo_wait_queue *from_waitq,49 struct vdo_wait_queue *to_waitq)50{51 /* If the source waitq is empty, there's nothing to do. */52 if (!vdo_waitq_has_waiters(from_waitq))53 return;54 55 if (vdo_waitq_has_waiters(to_waitq)) {56 /*57 * Both are non-empty. Splice the two circular lists together58 * by swapping the next (head) pointers in the list tails.59 */60 struct vdo_waiter *from_head = from_waitq->last_waiter->next_waiter;61 struct vdo_waiter *to_head = to_waitq->last_waiter->next_waiter;62 63 to_waitq->last_waiter->next_waiter = from_head;64 from_waitq->last_waiter->next_waiter = to_head;65 }66 67 to_waitq->last_waiter = from_waitq->last_waiter;68 to_waitq->length += from_waitq->length;69 vdo_waitq_init(from_waitq);70}71 72/**73 * vdo_waitq_notify_all_waiters() - Notify all the entries waiting in a waitq.74 * @waitq: The vdo_wait_queue containing the waiters to notify.75 * @callback: The function to call to notify each waiter, or NULL to invoke the callback field76 * registered in each waiter.77 * @context: The context to pass to the callback function.78 *79 * Notifies all the entries waiting in a waitq to continue execution by invoking a callback80 * function on each of them in turn. The waitq is copied and emptied before invoking any callbacks,81 * and only the waiters that were in the waitq at the start of the call will be notified.82 */83void vdo_waitq_notify_all_waiters(struct vdo_wait_queue *waitq,84 vdo_waiter_callback_fn callback, void *context)85{86 /*87 * Copy and empty the waitq first, avoiding the possibility of an infinite88 * loop if entries are returned to the waitq by the callback function.89 */90 struct vdo_wait_queue waiters;91 92 vdo_waitq_init(&waiters);93 vdo_waitq_transfer_all_waiters(waitq, &waiters);94 95 /* Drain the copied waitq, invoking the callback on every entry. */96 while (vdo_waitq_has_waiters(&waiters))97 vdo_waitq_notify_next_waiter(&waiters, callback, context);98}99 100/**101 * vdo_waitq_get_first_waiter() - Return the waiter that is at the head end of a waitq.102 * @waitq: The vdo_wait_queue from which to get the first waiter.103 *104 * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty.105 */106struct vdo_waiter *vdo_waitq_get_first_waiter(const struct vdo_wait_queue *waitq)107{108 struct vdo_waiter *last_waiter = waitq->last_waiter;109 110 if (last_waiter == NULL) {111 /* There are no waiters, so we're done. */112 return NULL;113 }114 115 /* The waitq is circular, so the last entry links to the head of the waitq. */116 return last_waiter->next_waiter;117}118 119/**120 * vdo_waitq_dequeue_matching_waiters() - Remove all waiters that match based on the specified121 * matching method and append them to a vdo_wait_queue.122 * @waitq: The vdo_wait_queue to process.123 * @waiter_match: The method to determine matching.124 * @match_context: Contextual info for the match method.125 * @matched_waitq: A wait_waitq to store matches.126 */127void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq,128 vdo_waiter_match_fn waiter_match,129 void *match_context,130 struct vdo_wait_queue *matched_waitq)131{132 struct vdo_wait_queue iteration_waitq;133 134 vdo_waitq_init(&iteration_waitq);135 vdo_waitq_transfer_all_waiters(waitq, &iteration_waitq);136 137 while (vdo_waitq_has_waiters(&iteration_waitq)) {138 struct vdo_waiter *waiter = vdo_waitq_dequeue_waiter(&iteration_waitq);139 140 vdo_waitq_enqueue_waiter((waiter_match(waiter, match_context) ?141 matched_waitq : waitq), waiter);142 }143}144 145/**146 * vdo_waitq_dequeue_waiter() - Remove the first (oldest) waiter from a waitq.147 * @waitq: The vdo_wait_queue from which to remove the first entry.148 *149 * The caller will be responsible for waking the waiter by continuing its150 * execution appropriately.151 *152 * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty.153 */154struct vdo_waiter *vdo_waitq_dequeue_waiter(struct vdo_wait_queue *waitq)155{156 struct vdo_waiter *first_waiter = vdo_waitq_get_first_waiter(waitq);157 struct vdo_waiter *last_waiter = waitq->last_waiter;158 159 if (first_waiter == NULL)160 return NULL;161 162 if (first_waiter == last_waiter) {163 /* The waitq has a single entry, so empty it by nulling the tail. */164 waitq->last_waiter = NULL;165 } else {166 /*167 * The waitq has multiple waiters, so splice the first waiter out168 * of the circular waitq.169 */170 last_waiter->next_waiter = first_waiter->next_waiter;171 }172 173 /* The waiter is no longer in a waitq. */174 first_waiter->next_waiter = NULL;175 waitq->length -= 1;176 177 return first_waiter;178}179 180/**181 * vdo_waitq_notify_next_waiter() - Notify the next entry waiting in a waitq.182 * @waitq: The vdo_wait_queue containing the waiter to notify.183 * @callback: The function to call to notify the waiter, or NULL to invoke the callback field184 * registered in the waiter.185 * @context: The context to pass to the callback function.186 *187 * Notifies the next entry waiting in a waitq to continue execution by invoking a callback function188 * on it after removing it from the waitq.189 *190 * Return: true if there was a waiter in the waitq.191 */192bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq,193 vdo_waiter_callback_fn callback, void *context)194{195 struct vdo_waiter *waiter = vdo_waitq_dequeue_waiter(waitq);196 197 if (waiter == NULL)198 return false;199 200 if (callback == NULL)201 callback = waiter->callback;202 callback(waiter, context);203 204 return true;205}206