945 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2014 Facebook. All rights reserved.4 *5 * This file is released under the GPL.6 */7 8#include <linux/device-mapper.h>9 10#include <linux/module.h>11#include <linux/init.h>12#include <linux/blkdev.h>13#include <linux/bio.h>14#include <linux/dax.h>15#include <linux/slab.h>16#include <linux/kthread.h>17#include <linux/freezer.h>18#include <linux/uio.h>19 20#define DM_MSG_PREFIX "log-writes"21 22/*23 * This target will sequentially log all writes to the target device onto the24 * log device. This is helpful for replaying writes to check for fs consistency25 * at all times. This target provides a mechanism to mark specific events to26 * check data at a later time. So for example you would:27 *28 * write data29 * fsync30 * dmsetup message /dev/whatever mark mymark31 * unmount /mnt/test32 *33 * Then replay the log up to mymark and check the contents of the replay to34 * verify it matches what was written.35 *36 * We log writes only after they have been flushed, this makes the log describe37 * close to the order in which the data hits the actual disk, not its cache. So38 * for example the following sequence (W means write, C means complete)39 *40 * Wa,Wb,Wc,Cc,Ca,FLUSH,FUAd,Cb,CFLUSH,CFUAd41 *42 * Would result in the log looking like this:43 *44 * c,a,b,flush,fuad,<other writes>,<next flush>45 *46 * This is meant to help expose problems where file systems do not properly wait47 * on data being written before invoking a FLUSH. FUA bypasses cache so once it48 * completes it is added to the log as it should be on disk.49 *50 * We treat DISCARDs as if they don't bypass cache so that they are logged in51 * order of completion along with the normal writes. If we didn't do it this52 * way we would process all the discards first and then write all the data, when53 * in fact we want to do the data and the discard in the order that they54 * completed.55 */56#define LOG_FLUSH_FLAG (1 << 0)57#define LOG_FUA_FLAG (1 << 1)58#define LOG_DISCARD_FLAG (1 << 2)59#define LOG_MARK_FLAG (1 << 3)60#define LOG_METADATA_FLAG (1 << 4)61 62#define WRITE_LOG_VERSION 1ULL63#define WRITE_LOG_MAGIC 0x6a736677736872ULL64#define WRITE_LOG_SUPER_SECTOR 065 66/*67 * The disk format for this is braindead simple.68 *69 * At byte 0 we have our super, followed by the following sequence for70 * nr_entries:71 *72 * [ 1 sector ][ entry->nr_sectors ]73 * [log_write_entry][ data written ]74 *75 * The log_write_entry takes up a full sector so we can have arbitrary length76 * marks and it leaves us room for extra content in the future.77 */78 79/*80 * Basic info about the log for userspace.81 */82struct log_write_super {83 __le64 magic;84 __le64 version;85 __le64 nr_entries;86 __le32 sectorsize;87};88 89/*90 * sector - the sector we wrote.91 * nr_sectors - the number of sectors we wrote.92 * flags - flags for this log entry.93 * data_len - the size of the data in this log entry, this is for private log94 * entry stuff, the MARK data provided by userspace for example.95 */96struct log_write_entry {97 __le64 sector;98 __le64 nr_sectors;99 __le64 flags;100 __le64 data_len;101};102 103struct log_writes_c {104 struct dm_dev *dev;105 struct dm_dev *logdev;106 u64 logged_entries;107 u32 sectorsize;108 u32 sectorshift;109 atomic_t io_blocks;110 atomic_t pending_blocks;111 sector_t next_sector;112 sector_t end_sector;113 bool logging_enabled;114 bool device_supports_discard;115 spinlock_t blocks_lock;116 struct list_head unflushed_blocks;117 struct list_head logging_blocks;118 wait_queue_head_t wait;119 struct task_struct *log_kthread;120 struct completion super_done;121};122 123struct pending_block {124 int vec_cnt;125 u64 flags;126 sector_t sector;127 sector_t nr_sectors;128 char *data;129 u32 datalen;130 struct list_head list;131 struct bio_vec vecs[];132};133 134struct per_bio_data {135 struct pending_block *block;136};137 138static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,139 sector_t sectors)140{141 return sectors >> (lc->sectorshift - SECTOR_SHIFT);142}143 144static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,145 sector_t sectors)146{147 return sectors << (lc->sectorshift - SECTOR_SHIFT);148}149 150static void put_pending_block(struct log_writes_c *lc)151{152 if (atomic_dec_and_test(&lc->pending_blocks)) {153 smp_mb__after_atomic();154 if (waitqueue_active(&lc->wait))155 wake_up(&lc->wait);156 }157}158 159static void put_io_block(struct log_writes_c *lc)160{161 if (atomic_dec_and_test(&lc->io_blocks)) {162 smp_mb__after_atomic();163 if (waitqueue_active(&lc->wait))164 wake_up(&lc->wait);165 }166}167 168static void log_end_io(struct bio *bio)169{170 struct log_writes_c *lc = bio->bi_private;171 172 if (bio->bi_status) {173 unsigned long flags;174 175 DMERR("Error writing log block, error=%d", bio->bi_status);176 spin_lock_irqsave(&lc->blocks_lock, flags);177 lc->logging_enabled = false;178 spin_unlock_irqrestore(&lc->blocks_lock, flags);179 }180 181 bio_free_pages(bio);182 put_io_block(lc);183 bio_put(bio);184}185 186static void log_end_super(struct bio *bio)187{188 struct log_writes_c *lc = bio->bi_private;189 190 complete(&lc->super_done);191 log_end_io(bio);192}193 194/*195 * Meant to be called if there is an error, it will free all the pages196 * associated with the block.197 */198static void free_pending_block(struct log_writes_c *lc,199 struct pending_block *block)200{201 int i;202 203 for (i = 0; i < block->vec_cnt; i++) {204 if (block->vecs[i].bv_page)205 __free_page(block->vecs[i].bv_page);206 }207 kfree(block->data);208 kfree(block);209 put_pending_block(lc);210}211 212static int write_metadata(struct log_writes_c *lc, void *entry,213 size_t entrylen, void *data, size_t datalen,214 sector_t sector)215{216 struct bio *bio;217 struct page *page;218 void *ptr;219 size_t ret;220 221 bio = bio_alloc(lc->logdev->bdev, 1, REQ_OP_WRITE, GFP_KERNEL);222 bio->bi_iter.bi_size = 0;223 bio->bi_iter.bi_sector = sector;224 bio->bi_end_io = (sector == WRITE_LOG_SUPER_SECTOR) ?225 log_end_super : log_end_io;226 bio->bi_private = lc;227 228 page = alloc_page(GFP_KERNEL);229 if (!page) {230 DMERR("Couldn't alloc log page");231 bio_put(bio);232 goto error;233 }234 235 ptr = kmap_local_page(page);236 memcpy(ptr, entry, entrylen);237 if (datalen)238 memcpy(ptr + entrylen, data, datalen);239 memset(ptr + entrylen + datalen, 0,240 lc->sectorsize - entrylen - datalen);241 kunmap_local(ptr);242 243 ret = bio_add_page(bio, page, lc->sectorsize, 0);244 if (ret != lc->sectorsize) {245 DMERR("Couldn't add page to the log block");246 goto error_bio;247 }248 submit_bio(bio);249 return 0;250error_bio:251 bio_put(bio);252 __free_page(page);253error:254 put_io_block(lc);255 return -1;256}257 258static int write_inline_data(struct log_writes_c *lc, void *entry,259 size_t entrylen, void *data, size_t datalen,260 sector_t sector)261{262 int bio_pages, pg_datalen, pg_sectorlen, i;263 struct page *page;264 struct bio *bio;265 size_t ret;266 void *ptr;267 268 while (datalen) {269 bio_pages = bio_max_segs(DIV_ROUND_UP(datalen, PAGE_SIZE));270 271 atomic_inc(&lc->io_blocks);272 273 bio = bio_alloc(lc->logdev->bdev, bio_pages, REQ_OP_WRITE,274 GFP_KERNEL);275 bio->bi_iter.bi_size = 0;276 bio->bi_iter.bi_sector = sector;277 bio->bi_end_io = log_end_io;278 bio->bi_private = lc;279 280 for (i = 0; i < bio_pages; i++) {281 pg_datalen = min_t(int, datalen, PAGE_SIZE);282 pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize);283 284 page = alloc_page(GFP_KERNEL);285 if (!page) {286 DMERR("Couldn't alloc inline data page");287 goto error_bio;288 }289 290 ptr = kmap_local_page(page);291 memcpy(ptr, data, pg_datalen);292 if (pg_sectorlen > pg_datalen)293 memset(ptr + pg_datalen, 0, pg_sectorlen - pg_datalen);294 kunmap_local(ptr);295 296 ret = bio_add_page(bio, page, pg_sectorlen, 0);297 if (ret != pg_sectorlen) {298 DMERR("Couldn't add page of inline data");299 __free_page(page);300 goto error_bio;301 }302 303 datalen -= pg_datalen;304 data += pg_datalen;305 }306 submit_bio(bio);307 308 sector += bio_pages * PAGE_SECTORS;309 }310 return 0;311error_bio:312 bio_free_pages(bio);313 bio_put(bio);314 put_io_block(lc);315 return -1;316}317 318static int log_one_block(struct log_writes_c *lc,319 struct pending_block *block, sector_t sector)320{321 struct bio *bio;322 struct log_write_entry entry;323 size_t metadatalen, ret;324 int i;325 326 entry.sector = cpu_to_le64(block->sector);327 entry.nr_sectors = cpu_to_le64(block->nr_sectors);328 entry.flags = cpu_to_le64(block->flags);329 entry.data_len = cpu_to_le64(block->datalen);330 331 metadatalen = (block->flags & LOG_MARK_FLAG) ? block->datalen : 0;332 if (write_metadata(lc, &entry, sizeof(entry), block->data,333 metadatalen, sector)) {334 free_pending_block(lc, block);335 return -1;336 }337 338 sector += dev_to_bio_sectors(lc, 1);339 340 if (block->datalen && metadatalen == 0) {341 if (write_inline_data(lc, &entry, sizeof(entry), block->data,342 block->datalen, sector)) {343 free_pending_block(lc, block);344 return -1;345 }346 /* we don't support both inline data & bio data */347 goto out;348 }349 350 if (!block->vec_cnt)351 goto out;352 353 atomic_inc(&lc->io_blocks);354 bio = bio_alloc(lc->logdev->bdev, bio_max_segs(block->vec_cnt),355 REQ_OP_WRITE, GFP_KERNEL);356 bio->bi_iter.bi_size = 0;357 bio->bi_iter.bi_sector = sector;358 bio->bi_end_io = log_end_io;359 bio->bi_private = lc;360 361 for (i = 0; i < block->vec_cnt; i++) {362 /*363 * The page offset is always 0 because we allocate a new page364 * for every bvec in the original bio for simplicity sake.365 */366 ret = bio_add_page(bio, block->vecs[i].bv_page,367 block->vecs[i].bv_len, 0);368 if (ret != block->vecs[i].bv_len) {369 atomic_inc(&lc->io_blocks);370 submit_bio(bio);371 bio = bio_alloc(lc->logdev->bdev,372 bio_max_segs(block->vec_cnt - i),373 REQ_OP_WRITE, GFP_KERNEL);374 bio->bi_iter.bi_size = 0;375 bio->bi_iter.bi_sector = sector;376 bio->bi_end_io = log_end_io;377 bio->bi_private = lc;378 379 ret = bio_add_page(bio, block->vecs[i].bv_page,380 block->vecs[i].bv_len, 0);381 if (ret != block->vecs[i].bv_len) {382 DMERR("Couldn't add page on new bio?");383 bio_put(bio);384 goto error;385 }386 }387 sector += block->vecs[i].bv_len >> SECTOR_SHIFT;388 }389 submit_bio(bio);390out:391 kfree(block->data);392 kfree(block);393 put_pending_block(lc);394 return 0;395error:396 free_pending_block(lc, block);397 put_io_block(lc);398 return -1;399}400 401static int log_super(struct log_writes_c *lc)402{403 struct log_write_super super;404 405 super.magic = cpu_to_le64(WRITE_LOG_MAGIC);406 super.version = cpu_to_le64(WRITE_LOG_VERSION);407 super.nr_entries = cpu_to_le64(lc->logged_entries);408 super.sectorsize = cpu_to_le32(lc->sectorsize);409 410 if (write_metadata(lc, &super, sizeof(super), NULL, 0,411 WRITE_LOG_SUPER_SECTOR)) {412 DMERR("Couldn't write super");413 return -1;414 }415 416 /*417 * Super sector should be writen in-order, otherwise the418 * nr_entries could be rewritten incorrectly by an old bio.419 */420 wait_for_completion_io(&lc->super_done);421 422 return 0;423}424 425static inline sector_t logdev_last_sector(struct log_writes_c *lc)426{427 return bdev_nr_sectors(lc->logdev->bdev);428}429 430static int log_writes_kthread(void *arg)431{432 struct log_writes_c *lc = arg;433 sector_t sector = 0;434 435 while (!kthread_should_stop()) {436 bool super = false;437 bool logging_enabled;438 struct pending_block *block = NULL;439 int ret;440 441 spin_lock_irq(&lc->blocks_lock);442 if (!list_empty(&lc->logging_blocks)) {443 block = list_first_entry(&lc->logging_blocks,444 struct pending_block, list);445 list_del_init(&block->list);446 if (!lc->logging_enabled)447 goto next;448 449 sector = lc->next_sector;450 if (!(block->flags & LOG_DISCARD_FLAG))451 lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);452 lc->next_sector += dev_to_bio_sectors(lc, 1);453 454 /*455 * Apparently the size of the device may not be known456 * right away, so handle this properly.457 */458 if (!lc->end_sector)459 lc->end_sector = logdev_last_sector(lc);460 if (lc->end_sector &&461 lc->next_sector >= lc->end_sector) {462 DMERR("Ran out of space on the logdev");463 lc->logging_enabled = false;464 goto next;465 }466 lc->logged_entries++;467 atomic_inc(&lc->io_blocks);468 469 super = (block->flags & (LOG_FUA_FLAG | LOG_MARK_FLAG));470 if (super)471 atomic_inc(&lc->io_blocks);472 }473next:474 logging_enabled = lc->logging_enabled;475 spin_unlock_irq(&lc->blocks_lock);476 if (block) {477 if (logging_enabled) {478 ret = log_one_block(lc, block, sector);479 if (!ret && super)480 ret = log_super(lc);481 if (ret) {482 spin_lock_irq(&lc->blocks_lock);483 lc->logging_enabled = false;484 spin_unlock_irq(&lc->blocks_lock);485 }486 } else487 free_pending_block(lc, block);488 continue;489 }490 491 if (!try_to_freeze()) {492 set_current_state(TASK_INTERRUPTIBLE);493 if (!kthread_should_stop() &&494 list_empty(&lc->logging_blocks))495 schedule();496 __set_current_state(TASK_RUNNING);497 }498 }499 return 0;500}501 502/*503 * Construct a log-writes mapping:504 * log-writes <dev_path> <log_dev_path>505 */506static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)507{508 struct log_writes_c *lc;509 struct dm_arg_set as;510 const char *devname, *logdevname;511 int ret;512 513 as.argc = argc;514 as.argv = argv;515 516 if (argc < 2) {517 ti->error = "Invalid argument count";518 return -EINVAL;519 }520 521 lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);522 if (!lc) {523 ti->error = "Cannot allocate context";524 return -ENOMEM;525 }526 spin_lock_init(&lc->blocks_lock);527 INIT_LIST_HEAD(&lc->unflushed_blocks);528 INIT_LIST_HEAD(&lc->logging_blocks);529 init_waitqueue_head(&lc->wait);530 init_completion(&lc->super_done);531 atomic_set(&lc->io_blocks, 0);532 atomic_set(&lc->pending_blocks, 0);533 534 devname = dm_shift_arg(&as);535 ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev);536 if (ret) {537 ti->error = "Device lookup failed";538 goto bad;539 }540 541 logdevname = dm_shift_arg(&as);542 ret = dm_get_device(ti, logdevname, dm_table_get_mode(ti->table),543 &lc->logdev);544 if (ret) {545 ti->error = "Log device lookup failed";546 dm_put_device(ti, lc->dev);547 goto bad;548 }549 550 lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);551 lc->sectorshift = ilog2(lc->sectorsize);552 lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");553 if (IS_ERR(lc->log_kthread)) {554 ret = PTR_ERR(lc->log_kthread);555 ti->error = "Couldn't alloc kthread";556 dm_put_device(ti, lc->dev);557 dm_put_device(ti, lc->logdev);558 goto bad;559 }560 561 /*562 * next_sector is in 512b sectors to correspond to what bi_sector expects.563 * The super starts at sector 0, and the next_sector is the next logical564 * one based on the sectorsize of the device.565 */566 lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;567 lc->logging_enabled = true;568 lc->end_sector = logdev_last_sector(lc);569 lc->device_supports_discard = true;570 571 ti->num_flush_bios = 1;572 ti->flush_supported = true;573 ti->num_discard_bios = 1;574 ti->discards_supported = true;575 ti->per_io_data_size = sizeof(struct per_bio_data);576 ti->private = lc;577 return 0;578 579bad:580 kfree(lc);581 return ret;582}583 584static int log_mark(struct log_writes_c *lc, char *data)585{586 struct pending_block *block;587 size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);588 589 block = kzalloc(sizeof(struct pending_block), GFP_KERNEL);590 if (!block) {591 DMERR("Error allocating pending block");592 return -ENOMEM;593 }594 595 block->data = kstrndup(data, maxsize - 1, GFP_KERNEL);596 if (!block->data) {597 DMERR("Error copying mark data");598 kfree(block);599 return -ENOMEM;600 }601 atomic_inc(&lc->pending_blocks);602 block->datalen = strlen(block->data);603 block->flags |= LOG_MARK_FLAG;604 spin_lock_irq(&lc->blocks_lock);605 list_add_tail(&block->list, &lc->logging_blocks);606 spin_unlock_irq(&lc->blocks_lock);607 wake_up_process(lc->log_kthread);608 return 0;609}610 611static void log_writes_dtr(struct dm_target *ti)612{613 struct log_writes_c *lc = ti->private;614 615 spin_lock_irq(&lc->blocks_lock);616 list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);617 spin_unlock_irq(&lc->blocks_lock);618 619 /*620 * This is just nice to have since it'll update the super to include the621 * unflushed blocks, if it fails we don't really care.622 */623 log_mark(lc, "dm-log-writes-end");624 wake_up_process(lc->log_kthread);625 wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&626 !atomic_read(&lc->pending_blocks));627 kthread_stop(lc->log_kthread);628 629 WARN_ON(!list_empty(&lc->logging_blocks));630 WARN_ON(!list_empty(&lc->unflushed_blocks));631 dm_put_device(ti, lc->dev);632 dm_put_device(ti, lc->logdev);633 kfree(lc);634}635 636static void normal_map_bio(struct dm_target *ti, struct bio *bio)637{638 struct log_writes_c *lc = ti->private;639 640 bio_set_dev(bio, lc->dev->bdev);641}642 643static int log_writes_map(struct dm_target *ti, struct bio *bio)644{645 struct log_writes_c *lc = ti->private;646 struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));647 struct pending_block *block;648 struct bvec_iter iter;649 struct bio_vec bv;650 size_t alloc_size;651 int i = 0;652 bool flush_bio = (bio->bi_opf & REQ_PREFLUSH);653 bool fua_bio = (bio->bi_opf & REQ_FUA);654 bool discard_bio = (bio_op(bio) == REQ_OP_DISCARD);655 bool meta_bio = (bio->bi_opf & REQ_META);656 657 pb->block = NULL;658 659 /* Don't bother doing anything if logging has been disabled */660 if (!lc->logging_enabled)661 goto map_bio;662 663 /*664 * Map reads as normal.665 */666 if (bio_data_dir(bio) == READ)667 goto map_bio;668 669 /* No sectors and not a flush? Don't care */670 if (!bio_sectors(bio) && !flush_bio)671 goto map_bio;672 673 /*674 * Discards will have bi_size set but there's no actual data, so just675 * allocate the size of the pending block.676 */677 if (discard_bio)678 alloc_size = sizeof(struct pending_block);679 else680 alloc_size = struct_size(block, vecs, bio_segments(bio));681 682 block = kzalloc(alloc_size, GFP_NOIO);683 if (!block) {684 DMERR("Error allocating pending block");685 spin_lock_irq(&lc->blocks_lock);686 lc->logging_enabled = false;687 spin_unlock_irq(&lc->blocks_lock);688 return DM_MAPIO_KILL;689 }690 INIT_LIST_HEAD(&block->list);691 pb->block = block;692 atomic_inc(&lc->pending_blocks);693 694 if (flush_bio)695 block->flags |= LOG_FLUSH_FLAG;696 if (fua_bio)697 block->flags |= LOG_FUA_FLAG;698 if (discard_bio)699 block->flags |= LOG_DISCARD_FLAG;700 if (meta_bio)701 block->flags |= LOG_METADATA_FLAG;702 703 block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);704 block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));705 706 /* We don't need the data, just submit */707 if (discard_bio) {708 WARN_ON(flush_bio || fua_bio);709 if (lc->device_supports_discard)710 goto map_bio;711 bio_endio(bio);712 return DM_MAPIO_SUBMITTED;713 }714 715 /* Flush bio, splice the unflushed blocks onto this list and submit */716 if (flush_bio && !bio_sectors(bio)) {717 spin_lock_irq(&lc->blocks_lock);718 list_splice_init(&lc->unflushed_blocks, &block->list);719 spin_unlock_irq(&lc->blocks_lock);720 goto map_bio;721 }722 723 /*724 * We will write this bio somewhere else way later so we need to copy725 * the actual contents into new pages so we know the data will always be726 * there.727 *728 * We do this because this could be a bio from O_DIRECT in which case we729 * can't just hold onto the page until some later point, we have to730 * manually copy the contents.731 */732 bio_for_each_segment(bv, bio, iter) {733 struct page *page;734 void *dst;735 736 page = alloc_page(GFP_NOIO);737 if (!page) {738 DMERR("Error allocing page");739 free_pending_block(lc, block);740 spin_lock_irq(&lc->blocks_lock);741 lc->logging_enabled = false;742 spin_unlock_irq(&lc->blocks_lock);743 return DM_MAPIO_KILL;744 }745 746 dst = kmap_local_page(page);747 memcpy_from_bvec(dst, &bv);748 kunmap_local(dst);749 block->vecs[i].bv_page = page;750 block->vecs[i].bv_len = bv.bv_len;751 block->vec_cnt++;752 i++;753 }754 755 /* Had a flush with data in it, weird */756 if (flush_bio) {757 spin_lock_irq(&lc->blocks_lock);758 list_splice_init(&lc->unflushed_blocks, &block->list);759 spin_unlock_irq(&lc->blocks_lock);760 }761map_bio:762 normal_map_bio(ti, bio);763 return DM_MAPIO_REMAPPED;764}765 766static int normal_end_io(struct dm_target *ti, struct bio *bio,767 blk_status_t *error)768{769 struct log_writes_c *lc = ti->private;770 struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));771 772 if (bio_data_dir(bio) == WRITE && pb->block) {773 struct pending_block *block = pb->block;774 unsigned long flags;775 776 spin_lock_irqsave(&lc->blocks_lock, flags);777 if (block->flags & LOG_FLUSH_FLAG) {778 list_splice_tail_init(&block->list, &lc->logging_blocks);779 list_add_tail(&block->list, &lc->logging_blocks);780 wake_up_process(lc->log_kthread);781 } else if (block->flags & LOG_FUA_FLAG) {782 list_add_tail(&block->list, &lc->logging_blocks);783 wake_up_process(lc->log_kthread);784 } else785 list_add_tail(&block->list, &lc->unflushed_blocks);786 spin_unlock_irqrestore(&lc->blocks_lock, flags);787 }788 789 return DM_ENDIO_DONE;790}791 792/*793 * INFO format: <logged entries> <highest allocated sector>794 */795static void log_writes_status(struct dm_target *ti, status_type_t type,796 unsigned int status_flags, char *result,797 unsigned int maxlen)798{799 unsigned int sz = 0;800 struct log_writes_c *lc = ti->private;801 802 switch (type) {803 case STATUSTYPE_INFO:804 DMEMIT("%llu %llu", lc->logged_entries,805 (unsigned long long)lc->next_sector - 1);806 if (!lc->logging_enabled)807 DMEMIT(" logging_disabled");808 break;809 810 case STATUSTYPE_TABLE:811 DMEMIT("%s %s", lc->dev->name, lc->logdev->name);812 break;813 814 case STATUSTYPE_IMA:815 *result = '\0';816 break;817 }818}819 820static int log_writes_prepare_ioctl(struct dm_target *ti,821 struct block_device **bdev)822{823 struct log_writes_c *lc = ti->private;824 struct dm_dev *dev = lc->dev;825 826 *bdev = dev->bdev;827 /*828 * Only pass ioctls through if the device sizes match exactly.829 */830 if (ti->len != bdev_nr_sectors(dev->bdev))831 return 1;832 return 0;833}834 835static int log_writes_iterate_devices(struct dm_target *ti,836 iterate_devices_callout_fn fn,837 void *data)838{839 struct log_writes_c *lc = ti->private;840 841 return fn(ti, lc->dev, 0, ti->len, data);842}843 844/*845 * Messages supported:846 * mark <mark data> - specify the marked data.847 */848static int log_writes_message(struct dm_target *ti, unsigned int argc, char **argv,849 char *result, unsigned int maxlen)850{851 int r = -EINVAL;852 struct log_writes_c *lc = ti->private;853 854 if (argc != 2) {855 DMWARN("Invalid log-writes message arguments, expect 2 arguments, got %d", argc);856 return r;857 }858 859 if (!strcasecmp(argv[0], "mark"))860 r = log_mark(lc, argv[1]);861 else862 DMWARN("Unrecognised log writes target message received: %s", argv[0]);863 864 return r;865}866 867static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limits)868{869 struct log_writes_c *lc = ti->private;870 871 if (!bdev_max_discard_sectors(lc->dev->bdev)) {872 lc->device_supports_discard = false;873 limits->discard_granularity = lc->sectorsize;874 limits->max_hw_discard_sectors = (UINT_MAX >> SECTOR_SHIFT);875 }876 limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);877 limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);878 limits->io_min = limits->physical_block_size;879 limits->dma_alignment = limits->logical_block_size - 1;880}881 882#if IS_ENABLED(CONFIG_FS_DAX)883static struct dax_device *log_writes_dax_pgoff(struct dm_target *ti,884 pgoff_t *pgoff)885{886 struct log_writes_c *lc = ti->private;887 888 *pgoff += (get_start_sect(lc->dev->bdev) >> PAGE_SECTORS_SHIFT);889 return lc->dev->dax_dev;890}891 892static long log_writes_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,893 long nr_pages, enum dax_access_mode mode, void **kaddr,894 pfn_t *pfn)895{896 struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);897 898 return dax_direct_access(dax_dev, pgoff, nr_pages, mode, kaddr, pfn);899}900 901static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff,902 size_t nr_pages)903{904 struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);905 906 return dax_zero_page_range(dax_dev, pgoff, nr_pages << PAGE_SHIFT);907}908 909static size_t log_writes_dax_recovery_write(struct dm_target *ti,910 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)911{912 struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);913 914 return dax_recovery_write(dax_dev, pgoff, addr, bytes, i);915}916 917#else918#define log_writes_dax_direct_access NULL919#define log_writes_dax_zero_page_range NULL920#define log_writes_dax_recovery_write NULL921#endif922 923static struct target_type log_writes_target = {924 .name = "log-writes",925 .version = {1, 1, 0},926 .module = THIS_MODULE,927 .ctr = log_writes_ctr,928 .dtr = log_writes_dtr,929 .map = log_writes_map,930 .end_io = normal_end_io,931 .status = log_writes_status,932 .prepare_ioctl = log_writes_prepare_ioctl,933 .message = log_writes_message,934 .iterate_devices = log_writes_iterate_devices,935 .io_hints = log_writes_io_hints,936 .direct_access = log_writes_dax_direct_access,937 .dax_zero_page_range = log_writes_dax_zero_page_range,938 .dax_recovery_write = log_writes_dax_recovery_write,939};940module_dm(log_writes);941 942MODULE_DESCRIPTION(DM_NAME " log writes target");943MODULE_AUTHOR("Josef Bacik <jbacik@fb.com>");944MODULE_LICENSE("GPL");945