663 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2011 Red Hat, Inc.4 *5 * This file is released under the GPL.6 */7#include "dm-block-manager.h"8#include "dm-persistent-data-internal.h"9 10#include <linux/dm-bufio.h>11#include <linux/crc32c.h>12#include <linux/module.h>13#include <linux/slab.h>14#include <linux/rwsem.h>15#include <linux/device-mapper.h>16#include <linux/stacktrace.h>17#include <linux/sched/task.h>18 19#define DM_MSG_PREFIX "block manager"20 21/*----------------------------------------------------------------*/22 23#ifdef CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING24 25/*26 * This is a read/write semaphore with a couple of differences.27 *28 * i) There is a restriction on the number of concurrent read locks that29 * may be held at once. This is just an implementation detail.30 *31 * ii) Recursive locking attempts are detected and return EINVAL. A stack32 * trace is also emitted for the previous lock acquisition.33 *34 * iii) Priority is given to write locks.35 */36#define MAX_HOLDERS 437#define MAX_STACK 1038 39struct stack_store {40 unsigned int nr_entries;41 unsigned long entries[MAX_STACK];42};43 44struct block_lock {45 spinlock_t lock;46 __s32 count;47 struct list_head waiters;48 struct task_struct *holders[MAX_HOLDERS];49 50#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING51 struct stack_store traces[MAX_HOLDERS];52#endif53};54 55struct waiter {56 struct list_head list;57 struct task_struct *task;58 int wants_write;59};60 61static unsigned int __find_holder(struct block_lock *lock,62 struct task_struct *task)63{64 unsigned int i;65 66 for (i = 0; i < MAX_HOLDERS; i++)67 if (lock->holders[i] == task)68 break;69 70 BUG_ON(i == MAX_HOLDERS);71 return i;72}73 74/* call this *after* you increment lock->count */75static void __add_holder(struct block_lock *lock, struct task_struct *task)76{77 unsigned int h = __find_holder(lock, NULL);78#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING79 struct stack_store *t;80#endif81 82 get_task_struct(task);83 lock->holders[h] = task;84 85#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING86 t = lock->traces + h;87 t->nr_entries = stack_trace_save(t->entries, MAX_STACK, 2);88#endif89}90 91/* call this *before* you decrement lock->count */92static void __del_holder(struct block_lock *lock, struct task_struct *task)93{94 unsigned int h = __find_holder(lock, task);95 96 lock->holders[h] = NULL;97 put_task_struct(task);98}99 100static int __check_holder(struct block_lock *lock)101{102 unsigned int i;103 104 for (i = 0; i < MAX_HOLDERS; i++) {105 if (lock->holders[i] == current) {106 DMERR("recursive lock detected in metadata");107#ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING108 DMERR("previously held here:");109 stack_trace_print(lock->traces[i].entries,110 lock->traces[i].nr_entries, 4);111 112 DMERR("subsequent acquisition attempted here:");113 dump_stack();114#endif115 return -EINVAL;116 }117 }118 119 return 0;120}121 122static void __wait(struct waiter *w)123{124 for (;;) {125 set_current_state(TASK_UNINTERRUPTIBLE);126 127 if (!w->task)128 break;129 130 schedule();131 }132 133 set_current_state(TASK_RUNNING);134}135 136static void __wake_waiter(struct waiter *w)137{138 struct task_struct *task;139 140 list_del(&w->list);141 task = w->task;142 smp_mb();143 w->task = NULL;144 wake_up_process(task);145}146 147/*148 * We either wake a few readers or a single writer.149 */150static void __wake_many(struct block_lock *lock)151{152 struct waiter *w, *tmp;153 154 BUG_ON(lock->count < 0);155 list_for_each_entry_safe(w, tmp, &lock->waiters, list) {156 if (lock->count >= MAX_HOLDERS)157 return;158 159 if (w->wants_write) {160 if (lock->count > 0)161 return; /* still read locked */162 163 lock->count = -1;164 __add_holder(lock, w->task);165 __wake_waiter(w);166 return;167 }168 169 lock->count++;170 __add_holder(lock, w->task);171 __wake_waiter(w);172 }173}174 175static void bl_init(struct block_lock *lock)176{177 int i;178 179 spin_lock_init(&lock->lock);180 lock->count = 0;181 INIT_LIST_HEAD(&lock->waiters);182 for (i = 0; i < MAX_HOLDERS; i++)183 lock->holders[i] = NULL;184}185 186static int __available_for_read(struct block_lock *lock)187{188 return lock->count >= 0 &&189 lock->count < MAX_HOLDERS &&190 list_empty(&lock->waiters);191}192 193static int bl_down_read(struct block_lock *lock)194{195 int r;196 struct waiter w;197 198 spin_lock(&lock->lock);199 r = __check_holder(lock);200 if (r) {201 spin_unlock(&lock->lock);202 return r;203 }204 205 if (__available_for_read(lock)) {206 lock->count++;207 __add_holder(lock, current);208 spin_unlock(&lock->lock);209 return 0;210 }211 212 get_task_struct(current);213 214 w.task = current;215 w.wants_write = 0;216 list_add_tail(&w.list, &lock->waiters);217 spin_unlock(&lock->lock);218 219 __wait(&w);220 put_task_struct(current);221 return 0;222}223 224static int bl_down_read_nonblock(struct block_lock *lock)225{226 int r;227 228 spin_lock(&lock->lock);229 r = __check_holder(lock);230 if (r)231 goto out;232 233 if (__available_for_read(lock)) {234 lock->count++;235 __add_holder(lock, current);236 r = 0;237 } else238 r = -EWOULDBLOCK;239 240out:241 spin_unlock(&lock->lock);242 return r;243}244 245static void bl_up_read(struct block_lock *lock)246{247 spin_lock(&lock->lock);248 BUG_ON(lock->count <= 0);249 __del_holder(lock, current);250 --lock->count;251 if (!list_empty(&lock->waiters))252 __wake_many(lock);253 spin_unlock(&lock->lock);254}255 256static int bl_down_write(struct block_lock *lock)257{258 int r;259 struct waiter w;260 261 spin_lock(&lock->lock);262 r = __check_holder(lock);263 if (r) {264 spin_unlock(&lock->lock);265 return r;266 }267 268 if (lock->count == 0 && list_empty(&lock->waiters)) {269 lock->count = -1;270 __add_holder(lock, current);271 spin_unlock(&lock->lock);272 return 0;273 }274 275 get_task_struct(current);276 w.task = current;277 w.wants_write = 1;278 279 /*280 * Writers given priority. We know there's only one mutator in the281 * system, so ignoring the ordering reversal.282 */283 list_add(&w.list, &lock->waiters);284 spin_unlock(&lock->lock);285 286 __wait(&w);287 put_task_struct(current);288 289 return 0;290}291 292static void bl_up_write(struct block_lock *lock)293{294 spin_lock(&lock->lock);295 __del_holder(lock, current);296 lock->count = 0;297 if (!list_empty(&lock->waiters))298 __wake_many(lock);299 spin_unlock(&lock->lock);300}301 302static void report_recursive_bug(dm_block_t b, int r)303{304 if (r == -EINVAL)305 DMERR("recursive acquisition of block %llu requested.",306 (unsigned long long) b);307}308 309#else /* !CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING */310 311#define bl_init(x) do { } while (0)312#define bl_down_read(x) 0313#define bl_down_read_nonblock(x) 0314#define bl_up_read(x) do { } while (0)315#define bl_down_write(x) 0316#define bl_up_write(x) do { } while (0)317#define report_recursive_bug(x, y) do { } while (0)318 319#endif /* CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING */320 321/*----------------------------------------------------------------*/322 323/*324 * Block manager is currently implemented using dm-bufio. struct325 * dm_block_manager and struct dm_block map directly onto a couple of326 * structs in the bufio interface. I want to retain the freedom to move327 * away from bufio in the future. So these structs are just cast within328 * this .c file, rather than making it through to the public interface.329 */330static struct dm_buffer *to_buffer(struct dm_block *b)331{332 return (struct dm_buffer *) b;333}334 335dm_block_t dm_block_location(struct dm_block *b)336{337 return dm_bufio_get_block_number(to_buffer(b));338}339EXPORT_SYMBOL_GPL(dm_block_location);340 341void *dm_block_data(struct dm_block *b)342{343 return dm_bufio_get_block_data(to_buffer(b));344}345EXPORT_SYMBOL_GPL(dm_block_data);346 347struct buffer_aux {348 const struct dm_block_validator *validator;349 int write_locked;350 351#ifdef CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING352 struct block_lock lock;353#endif354};355 356static void dm_block_manager_alloc_callback(struct dm_buffer *buf)357{358 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);359 360 aux->validator = NULL;361 bl_init(&aux->lock);362}363 364static void dm_block_manager_write_callback(struct dm_buffer *buf)365{366 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);367 368 if (aux->validator) {369 aux->validator->prepare_for_write(aux->validator, (struct dm_block *) buf,370 dm_bufio_get_block_size(dm_bufio_get_client(buf)));371 }372}373 374/*375 * -------------------------------------------------------------376 * Public interface377 *--------------------------------------------------------------378 */379struct dm_block_manager {380 struct dm_bufio_client *bufio;381 bool read_only:1;382};383 384struct dm_block_manager *dm_block_manager_create(struct block_device *bdev,385 unsigned int block_size,386 unsigned int max_held_per_thread)387{388 int r;389 struct dm_block_manager *bm;390 391 bm = kmalloc(sizeof(*bm), GFP_KERNEL);392 if (!bm) {393 r = -ENOMEM;394 goto bad;395 }396 397 bm->bufio = dm_bufio_client_create(bdev, block_size, max_held_per_thread,398 sizeof(struct buffer_aux),399 dm_block_manager_alloc_callback,400 dm_block_manager_write_callback,401 0);402 if (IS_ERR(bm->bufio)) {403 r = PTR_ERR(bm->bufio);404 kfree(bm);405 goto bad;406 }407 408 bm->read_only = false;409 410 return bm;411 412bad:413 return ERR_PTR(r);414}415EXPORT_SYMBOL_GPL(dm_block_manager_create);416 417void dm_block_manager_destroy(struct dm_block_manager *bm)418{419 dm_bufio_client_destroy(bm->bufio);420 kfree(bm);421}422EXPORT_SYMBOL_GPL(dm_block_manager_destroy);423 424void dm_block_manager_reset(struct dm_block_manager *bm)425{426 dm_bufio_client_reset(bm->bufio);427}428EXPORT_SYMBOL_GPL(dm_block_manager_reset);429 430unsigned int dm_bm_block_size(struct dm_block_manager *bm)431{432 return dm_bufio_get_block_size(bm->bufio);433}434EXPORT_SYMBOL_GPL(dm_bm_block_size);435 436dm_block_t dm_bm_nr_blocks(struct dm_block_manager *bm)437{438 return dm_bufio_get_device_size(bm->bufio);439}440 441static int dm_bm_validate_buffer(struct dm_block_manager *bm,442 struct dm_buffer *buf,443 struct buffer_aux *aux,444 const struct dm_block_validator *v)445{446 if (unlikely(!aux->validator)) {447 int r;448 449 if (!v)450 return 0;451 r = v->check(v, (struct dm_block *) buf, dm_bufio_get_block_size(bm->bufio));452 if (unlikely(r)) {453 DMERR_LIMIT("%s validator check failed for block %llu", v->name,454 (unsigned long long) dm_bufio_get_block_number(buf));455 return r;456 }457 aux->validator = v;458 } else {459 if (unlikely(aux->validator != v)) {460 DMERR_LIMIT("validator mismatch (old=%s vs new=%s) for block %llu",461 aux->validator->name, v ? v->name : "NULL",462 (unsigned long long) dm_bufio_get_block_number(buf));463 return -EINVAL;464 }465 }466 467 return 0;468}469int dm_bm_read_lock(struct dm_block_manager *bm, dm_block_t b,470 const struct dm_block_validator *v,471 struct dm_block **result)472{473 struct buffer_aux *aux;474 void *p;475 int r;476 477 p = dm_bufio_read(bm->bufio, b, (struct dm_buffer **) result);478 if (IS_ERR(p))479 return PTR_ERR(p);480 481 aux = dm_bufio_get_aux_data(to_buffer(*result));482 r = bl_down_read(&aux->lock);483 if (unlikely(r)) {484 dm_bufio_release(to_buffer(*result));485 report_recursive_bug(b, r);486 return r;487 }488 489 aux->write_locked = 0;490 491 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);492 if (unlikely(r)) {493 bl_up_read(&aux->lock);494 dm_bufio_release(to_buffer(*result));495 return r;496 }497 498 return 0;499}500EXPORT_SYMBOL_GPL(dm_bm_read_lock);501 502int dm_bm_write_lock(struct dm_block_manager *bm,503 dm_block_t b, const struct dm_block_validator *v,504 struct dm_block **result)505{506 struct buffer_aux *aux;507 void *p;508 int r;509 510 if (dm_bm_is_read_only(bm))511 return -EPERM;512 513 p = dm_bufio_read(bm->bufio, b, (struct dm_buffer **) result);514 if (IS_ERR(p))515 return PTR_ERR(p);516 517 aux = dm_bufio_get_aux_data(to_buffer(*result));518 r = bl_down_write(&aux->lock);519 if (r) {520 dm_bufio_release(to_buffer(*result));521 report_recursive_bug(b, r);522 return r;523 }524 525 aux->write_locked = 1;526 527 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);528 if (unlikely(r)) {529 bl_up_write(&aux->lock);530 dm_bufio_release(to_buffer(*result));531 return r;532 }533 534 return 0;535}536EXPORT_SYMBOL_GPL(dm_bm_write_lock);537 538int dm_bm_read_try_lock(struct dm_block_manager *bm,539 dm_block_t b, const struct dm_block_validator *v,540 struct dm_block **result)541{542 struct buffer_aux *aux;543 void *p;544 int r;545 546 p = dm_bufio_get(bm->bufio, b, (struct dm_buffer **) result);547 if (IS_ERR(p))548 return PTR_ERR(p);549 if (unlikely(!p))550 return -EWOULDBLOCK;551 552 aux = dm_bufio_get_aux_data(to_buffer(*result));553 r = bl_down_read_nonblock(&aux->lock);554 if (r < 0) {555 dm_bufio_release(to_buffer(*result));556 report_recursive_bug(b, r);557 return r;558 }559 aux->write_locked = 0;560 561 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);562 if (unlikely(r)) {563 bl_up_read(&aux->lock);564 dm_bufio_release(to_buffer(*result));565 return r;566 }567 568 return 0;569}570 571int dm_bm_write_lock_zero(struct dm_block_manager *bm,572 dm_block_t b, const struct dm_block_validator *v,573 struct dm_block **result)574{575 int r;576 struct buffer_aux *aux;577 void *p;578 579 if (dm_bm_is_read_only(bm))580 return -EPERM;581 582 p = dm_bufio_new(bm->bufio, b, (struct dm_buffer **) result);583 if (IS_ERR(p))584 return PTR_ERR(p);585 586 memset(p, 0, dm_bm_block_size(bm));587 588 aux = dm_bufio_get_aux_data(to_buffer(*result));589 r = bl_down_write(&aux->lock);590 if (r) {591 dm_bufio_release(to_buffer(*result));592 return r;593 }594 595 aux->write_locked = 1;596 aux->validator = v;597 598 return 0;599}600EXPORT_SYMBOL_GPL(dm_bm_write_lock_zero);601 602void dm_bm_unlock(struct dm_block *b)603{604 struct buffer_aux *aux = dm_bufio_get_aux_data(to_buffer(b));605 606 if (aux->write_locked) {607 dm_bufio_mark_buffer_dirty(to_buffer(b));608 bl_up_write(&aux->lock);609 } else610 bl_up_read(&aux->lock);611 612 dm_bufio_release(to_buffer(b));613}614EXPORT_SYMBOL_GPL(dm_bm_unlock);615 616int dm_bm_flush(struct dm_block_manager *bm)617{618 if (dm_bm_is_read_only(bm))619 return -EPERM;620 621 return dm_bufio_write_dirty_buffers(bm->bufio);622}623EXPORT_SYMBOL_GPL(dm_bm_flush);624 625void dm_bm_prefetch(struct dm_block_manager *bm, dm_block_t b)626{627 dm_bufio_prefetch(bm->bufio, b, 1);628}629 630bool dm_bm_is_read_only(struct dm_block_manager *bm)631{632 return bm ? bm->read_only : true;633}634EXPORT_SYMBOL_GPL(dm_bm_is_read_only);635 636void dm_bm_set_read_only(struct dm_block_manager *bm)637{638 if (bm)639 bm->read_only = true;640}641EXPORT_SYMBOL_GPL(dm_bm_set_read_only);642 643void dm_bm_set_read_write(struct dm_block_manager *bm)644{645 if (bm)646 bm->read_only = false;647}648EXPORT_SYMBOL_GPL(dm_bm_set_read_write);649 650u32 dm_bm_checksum(const void *data, size_t len, u32 init_xor)651{652 return crc32c(~(u32) 0, data, len) ^ init_xor;653}654EXPORT_SYMBOL_GPL(dm_bm_checksum);655 656/*----------------------------------------------------------------*/657 658MODULE_LICENSE("GPL");659MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>");660MODULE_DESCRIPTION("Immutable metadata library for dm");661 662/*----------------------------------------------------------------*/663