1330 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 1991, 1992 Linus Torvalds4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE5 * Copyright (C) 2016 - 2020 Christoph Hellwig6 */7 8#include <linux/init.h>9#include <linux/mm.h>10#include <linux/slab.h>11#include <linux/kmod.h>12#include <linux/major.h>13#include <linux/device_cgroup.h>14#include <linux/blkdev.h>15#include <linux/blk-integrity.h>16#include <linux/backing-dev.h>17#include <linux/module.h>18#include <linux/blkpg.h>19#include <linux/magic.h>20#include <linux/buffer_head.h>21#include <linux/swap.h>22#include <linux/writeback.h>23#include <linux/mount.h>24#include <linux/pseudo_fs.h>25#include <linux/uio.h>26#include <linux/namei.h>27#include <linux/security.h>28#include <linux/part_stat.h>29#include <linux/uaccess.h>30#include <linux/stat.h>31#include "../fs/internal.h"32#include "blk.h"33 34/* Should we allow writing to mounted block devices? */35static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED);36 37struct bdev_inode {38 struct block_device bdev;39 struct inode vfs_inode;40};41 42static inline struct bdev_inode *BDEV_I(struct inode *inode)43{44 return container_of(inode, struct bdev_inode, vfs_inode);45}46 47static inline struct inode *BD_INODE(struct block_device *bdev)48{49 return &container_of(bdev, struct bdev_inode, bdev)->vfs_inode;50}51 52struct block_device *I_BDEV(struct inode *inode)53{54 return &BDEV_I(inode)->bdev;55}56EXPORT_SYMBOL(I_BDEV);57 58struct block_device *file_bdev(struct file *bdev_file)59{60 return I_BDEV(bdev_file->f_mapping->host);61}62EXPORT_SYMBOL(file_bdev);63 64static void bdev_write_inode(struct block_device *bdev)65{66 struct inode *inode = BD_INODE(bdev);67 int ret;68 69 spin_lock(&inode->i_lock);70 while (inode->i_state & I_DIRTY) {71 spin_unlock(&inode->i_lock);72 ret = write_inode_now(inode, true);73 if (ret)74 pr_warn_ratelimited(75 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n",76 bdev, ret);77 spin_lock(&inode->i_lock);78 }79 spin_unlock(&inode->i_lock);80}81 82/* Kill _all_ buffers and pagecache , dirty or not.. */83static void kill_bdev(struct block_device *bdev)84{85 struct address_space *mapping = bdev->bd_mapping;86 87 if (mapping_empty(mapping))88 return;89 90 invalidate_bh_lrus();91 truncate_inode_pages(mapping, 0);92}93 94/* Invalidate clean unused buffers and pagecache. */95void invalidate_bdev(struct block_device *bdev)96{97 struct address_space *mapping = bdev->bd_mapping;98 99 if (mapping->nrpages) {100 invalidate_bh_lrus();101 lru_add_drain_all(); /* make sure all lru add caches are flushed */102 invalidate_mapping_pages(mapping, 0, -1);103 }104}105EXPORT_SYMBOL(invalidate_bdev);106 107/*108 * Drop all buffers & page cache for given bdev range. This function bails109 * with error if bdev has other exclusive owner (such as filesystem).110 */111int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,112 loff_t lstart, loff_t lend)113{114 /*115 * If we don't hold exclusive handle for the device, upgrade to it116 * while we discard the buffer cache to avoid discarding buffers117 * under live filesystem.118 */119 if (!(mode & BLK_OPEN_EXCL)) {120 int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL);121 if (err)122 goto invalidate;123 }124 125 truncate_inode_pages_range(bdev->bd_mapping, lstart, lend);126 if (!(mode & BLK_OPEN_EXCL))127 bd_abort_claiming(bdev, truncate_bdev_range);128 return 0;129 130invalidate:131 /*132 * Someone else has handle exclusively open. Try invalidating instead.133 * The 'end' argument is inclusive so the rounding is safe.134 */135 return invalidate_inode_pages2_range(bdev->bd_mapping,136 lstart >> PAGE_SHIFT,137 lend >> PAGE_SHIFT);138}139 140static void set_init_blocksize(struct block_device *bdev)141{142 unsigned int bsize = bdev_logical_block_size(bdev);143 loff_t size = i_size_read(BD_INODE(bdev));144 145 while (bsize < PAGE_SIZE) {146 if (size & bsize)147 break;148 bsize <<= 1;149 }150 BD_INODE(bdev)->i_blkbits = blksize_bits(bsize);151}152 153int set_blocksize(struct file *file, int size)154{155 struct inode *inode = file->f_mapping->host;156 struct block_device *bdev = I_BDEV(inode);157 158 /* Size must be a power of two, and between 512 and PAGE_SIZE */159 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))160 return -EINVAL;161 162 /* Size cannot be smaller than the size supported by the device */163 if (size < bdev_logical_block_size(bdev))164 return -EINVAL;165 166 if (!file->private_data)167 return -EINVAL;168 169 /* Don't change the size if it is same as current */170 if (inode->i_blkbits != blksize_bits(size)) {171 sync_blockdev(bdev);172 inode->i_blkbits = blksize_bits(size);173 kill_bdev(bdev);174 }175 return 0;176}177 178EXPORT_SYMBOL(set_blocksize);179 180int sb_set_blocksize(struct super_block *sb, int size)181{182 if (set_blocksize(sb->s_bdev_file, size))183 return 0;184 /* If we get here, we know size is power of two185 * and it's value is between 512 and PAGE_SIZE */186 sb->s_blocksize = size;187 sb->s_blocksize_bits = blksize_bits(size);188 return sb->s_blocksize;189}190 191EXPORT_SYMBOL(sb_set_blocksize);192 193int sb_min_blocksize(struct super_block *sb, int size)194{195 int minsize = bdev_logical_block_size(sb->s_bdev);196 if (size < minsize)197 size = minsize;198 return sb_set_blocksize(sb, size);199}200 201EXPORT_SYMBOL(sb_min_blocksize);202 203int sync_blockdev_nowait(struct block_device *bdev)204{205 if (!bdev)206 return 0;207 return filemap_flush(bdev->bd_mapping);208}209EXPORT_SYMBOL_GPL(sync_blockdev_nowait);210 211/*212 * Write out and wait upon all the dirty data associated with a block213 * device via its mapping. Does not take the superblock lock.214 */215int sync_blockdev(struct block_device *bdev)216{217 if (!bdev)218 return 0;219 return filemap_write_and_wait(bdev->bd_mapping);220}221EXPORT_SYMBOL(sync_blockdev);222 223int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend)224{225 return filemap_write_and_wait_range(bdev->bd_mapping,226 lstart, lend);227}228EXPORT_SYMBOL(sync_blockdev_range);229 230/**231 * bdev_freeze - lock a filesystem and force it into a consistent state232 * @bdev: blockdevice to lock233 *234 * If a superblock is found on this device, we take the s_umount semaphore235 * on it to make sure nobody unmounts until the snapshot creation is done.236 * The reference counter (bd_fsfreeze_count) guarantees that only the last237 * unfreeze process can unfreeze the frozen filesystem actually when multiple238 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and239 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze240 * actually.241 *242 * Return: On success zero is returned, negative error code on failure.243 */244int bdev_freeze(struct block_device *bdev)245{246 int error = 0;247 248 mutex_lock(&bdev->bd_fsfreeze_mutex);249 250 if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) {251 mutex_unlock(&bdev->bd_fsfreeze_mutex);252 return 0;253 }254 255 mutex_lock(&bdev->bd_holder_lock);256 if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) {257 error = bdev->bd_holder_ops->freeze(bdev);258 lockdep_assert_not_held(&bdev->bd_holder_lock);259 } else {260 mutex_unlock(&bdev->bd_holder_lock);261 error = sync_blockdev(bdev);262 }263 264 if (error)265 atomic_dec(&bdev->bd_fsfreeze_count);266 267 mutex_unlock(&bdev->bd_fsfreeze_mutex);268 return error;269}270EXPORT_SYMBOL(bdev_freeze);271 272/**273 * bdev_thaw - unlock filesystem274 * @bdev: blockdevice to unlock275 *276 * Unlocks the filesystem and marks it writeable again after bdev_freeze().277 *278 * Return: On success zero is returned, negative error code on failure.279 */280int bdev_thaw(struct block_device *bdev)281{282 int error = -EINVAL, nr_freeze;283 284 mutex_lock(&bdev->bd_fsfreeze_mutex);285 286 /*287 * If this returns < 0 it means that @bd_fsfreeze_count was288 * already 0 and no decrement was performed.289 */290 nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count);291 if (nr_freeze < 0)292 goto out;293 294 error = 0;295 if (nr_freeze > 0)296 goto out;297 298 mutex_lock(&bdev->bd_holder_lock);299 if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) {300 error = bdev->bd_holder_ops->thaw(bdev);301 lockdep_assert_not_held(&bdev->bd_holder_lock);302 } else {303 mutex_unlock(&bdev->bd_holder_lock);304 }305 306 if (error)307 atomic_inc(&bdev->bd_fsfreeze_count);308out:309 mutex_unlock(&bdev->bd_fsfreeze_mutex);310 return error;311}312EXPORT_SYMBOL(bdev_thaw);313 314/*315 * pseudo-fs316 */317 318static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock);319static struct kmem_cache *bdev_cachep __ro_after_init;320 321static struct inode *bdev_alloc_inode(struct super_block *sb)322{323 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL);324 325 if (!ei)326 return NULL;327 memset(&ei->bdev, 0, sizeof(ei->bdev));328 329 if (security_bdev_alloc(&ei->bdev)) {330 kmem_cache_free(bdev_cachep, ei);331 return NULL;332 }333 return &ei->vfs_inode;334}335 336static void bdev_free_inode(struct inode *inode)337{338 struct block_device *bdev = I_BDEV(inode);339 340 free_percpu(bdev->bd_stats);341 kfree(bdev->bd_meta_info);342 security_bdev_free(bdev);343 344 if (!bdev_is_partition(bdev)) {345 if (bdev->bd_disk && bdev->bd_disk->bdi)346 bdi_put(bdev->bd_disk->bdi);347 kfree(bdev->bd_disk);348 }349 350 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)351 blk_free_ext_minor(MINOR(bdev->bd_dev));352 353 kmem_cache_free(bdev_cachep, BDEV_I(inode));354}355 356static void init_once(void *data)357{358 struct bdev_inode *ei = data;359 360 inode_init_once(&ei->vfs_inode);361}362 363static void bdev_evict_inode(struct inode *inode)364{365 truncate_inode_pages_final(&inode->i_data);366 invalidate_inode_buffers(inode); /* is it needed here? */367 clear_inode(inode);368}369 370static const struct super_operations bdev_sops = {371 .statfs = simple_statfs,372 .alloc_inode = bdev_alloc_inode,373 .free_inode = bdev_free_inode,374 .drop_inode = generic_delete_inode,375 .evict_inode = bdev_evict_inode,376};377 378static int bd_init_fs_context(struct fs_context *fc)379{380 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);381 if (!ctx)382 return -ENOMEM;383 fc->s_iflags |= SB_I_CGROUPWB;384 ctx->ops = &bdev_sops;385 return 0;386}387 388static struct file_system_type bd_type = {389 .name = "bdev",390 .init_fs_context = bd_init_fs_context,391 .kill_sb = kill_anon_super,392};393 394struct super_block *blockdev_superblock __ro_after_init;395static struct vfsmount *blockdev_mnt __ro_after_init;396EXPORT_SYMBOL_GPL(blockdev_superblock);397 398void __init bdev_cache_init(void)399{400 int err;401 402 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),403 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|404 SLAB_ACCOUNT|SLAB_PANIC),405 init_once);406 err = register_filesystem(&bd_type);407 if (err)408 panic("Cannot register bdev pseudo-fs");409 blockdev_mnt = kern_mount(&bd_type);410 if (IS_ERR(blockdev_mnt))411 panic("Cannot create bdev pseudo-fs");412 blockdev_superblock = blockdev_mnt->mnt_sb; /* For writeback */413}414 415struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)416{417 struct block_device *bdev;418 struct inode *inode;419 420 inode = new_inode(blockdev_superblock);421 if (!inode)422 return NULL;423 inode->i_mode = S_IFBLK;424 inode->i_rdev = 0;425 inode->i_data.a_ops = &def_blk_aops;426 mapping_set_gfp_mask(&inode->i_data, GFP_USER);427 428 bdev = I_BDEV(inode);429 mutex_init(&bdev->bd_fsfreeze_mutex);430 spin_lock_init(&bdev->bd_size_lock);431 mutex_init(&bdev->bd_holder_lock);432 atomic_set(&bdev->__bd_flags, partno);433 bdev->bd_mapping = &inode->i_data;434 bdev->bd_queue = disk->queue;435 if (partno && bdev_test_flag(disk->part0, BD_HAS_SUBMIT_BIO))436 bdev_set_flag(bdev, BD_HAS_SUBMIT_BIO);437 bdev->bd_stats = alloc_percpu(struct disk_stats);438 if (!bdev->bd_stats) {439 iput(inode);440 return NULL;441 }442 bdev->bd_disk = disk;443 return bdev;444}445 446void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors)447{448 spin_lock(&bdev->bd_size_lock);449 i_size_write(BD_INODE(bdev), (loff_t)sectors << SECTOR_SHIFT);450 bdev->bd_nr_sectors = sectors;451 spin_unlock(&bdev->bd_size_lock);452}453 454void bdev_add(struct block_device *bdev, dev_t dev)455{456 struct inode *inode = BD_INODE(bdev);457 if (bdev_stable_writes(bdev))458 mapping_set_stable_writes(bdev->bd_mapping);459 bdev->bd_dev = dev;460 inode->i_rdev = dev;461 inode->i_ino = dev;462 insert_inode_hash(inode);463}464 465void bdev_unhash(struct block_device *bdev)466{467 remove_inode_hash(BD_INODE(bdev));468}469 470void bdev_drop(struct block_device *bdev)471{472 iput(BD_INODE(bdev));473}474 475long nr_blockdev_pages(void)476{477 struct inode *inode;478 long ret = 0;479 480 spin_lock(&blockdev_superblock->s_inode_list_lock);481 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)482 ret += inode->i_mapping->nrpages;483 spin_unlock(&blockdev_superblock->s_inode_list_lock);484 485 return ret;486}487 488/**489 * bd_may_claim - test whether a block device can be claimed490 * @bdev: block device of interest491 * @holder: holder trying to claim @bdev492 * @hops: holder ops493 *494 * Test whether @bdev can be claimed by @holder.495 *496 * RETURNS:497 * %true if @bdev can be claimed, %false otherwise.498 */499static bool bd_may_claim(struct block_device *bdev, void *holder,500 const struct blk_holder_ops *hops)501{502 struct block_device *whole = bdev_whole(bdev);503 504 lockdep_assert_held(&bdev_lock);505 506 if (bdev->bd_holder) {507 /*508 * The same holder can always re-claim.509 */510 if (bdev->bd_holder == holder) {511 if (WARN_ON_ONCE(bdev->bd_holder_ops != hops))512 return false;513 return true;514 }515 return false;516 }517 518 /*519 * If the whole devices holder is set to bd_may_claim, a partition on520 * the device is claimed, but not the whole device.521 */522 if (whole != bdev &&523 whole->bd_holder && whole->bd_holder != bd_may_claim)524 return false;525 return true;526}527 528/**529 * bd_prepare_to_claim - claim a block device530 * @bdev: block device of interest531 * @holder: holder trying to claim @bdev532 * @hops: holder ops.533 *534 * Claim @bdev. This function fails if @bdev is already claimed by another535 * holder and waits if another claiming is in progress. return, the caller536 * has ownership of bd_claiming and bd_holder[s].537 *538 * RETURNS:539 * 0 if @bdev can be claimed, -EBUSY otherwise.540 */541int bd_prepare_to_claim(struct block_device *bdev, void *holder,542 const struct blk_holder_ops *hops)543{544 struct block_device *whole = bdev_whole(bdev);545 546 if (WARN_ON_ONCE(!holder))547 return -EINVAL;548retry:549 mutex_lock(&bdev_lock);550 /* if someone else claimed, fail */551 if (!bd_may_claim(bdev, holder, hops)) {552 mutex_unlock(&bdev_lock);553 return -EBUSY;554 }555 556 /* if claiming is already in progress, wait for it to finish */557 if (whole->bd_claiming) {558 wait_queue_head_t *wq = __var_waitqueue(&whole->bd_claiming);559 DEFINE_WAIT(wait);560 561 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);562 mutex_unlock(&bdev_lock);563 schedule();564 finish_wait(wq, &wait);565 goto retry;566 }567 568 /* yay, all mine */569 whole->bd_claiming = holder;570 mutex_unlock(&bdev_lock);571 return 0;572}573EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */574 575static void bd_clear_claiming(struct block_device *whole, void *holder)576{577 lockdep_assert_held(&bdev_lock);578 /* tell others that we're done */579 BUG_ON(whole->bd_claiming != holder);580 whole->bd_claiming = NULL;581 wake_up_var(&whole->bd_claiming);582}583 584/**585 * bd_finish_claiming - finish claiming of a block device586 * @bdev: block device of interest587 * @holder: holder that has claimed @bdev588 * @hops: block device holder operations589 *590 * Finish exclusive open of a block device. Mark the device as exlusively591 * open by the holder and wake up all waiters for exclusive open to finish.592 */593static void bd_finish_claiming(struct block_device *bdev, void *holder,594 const struct blk_holder_ops *hops)595{596 struct block_device *whole = bdev_whole(bdev);597 598 mutex_lock(&bdev_lock);599 BUG_ON(!bd_may_claim(bdev, holder, hops));600 /*601 * Note that for a whole device bd_holders will be incremented twice,602 * and bd_holder will be set to bd_may_claim before being set to holder603 */604 whole->bd_holders++;605 whole->bd_holder = bd_may_claim;606 bdev->bd_holders++;607 mutex_lock(&bdev->bd_holder_lock);608 bdev->bd_holder = holder;609 bdev->bd_holder_ops = hops;610 mutex_unlock(&bdev->bd_holder_lock);611 bd_clear_claiming(whole, holder);612 mutex_unlock(&bdev_lock);613}614 615/**616 * bd_abort_claiming - abort claiming of a block device617 * @bdev: block device of interest618 * @holder: holder that has claimed @bdev619 *620 * Abort claiming of a block device when the exclusive open failed. This can be621 * also used when exclusive open is not actually desired and we just needed622 * to block other exclusive openers for a while.623 */624void bd_abort_claiming(struct block_device *bdev, void *holder)625{626 mutex_lock(&bdev_lock);627 bd_clear_claiming(bdev_whole(bdev), holder);628 mutex_unlock(&bdev_lock);629}630EXPORT_SYMBOL(bd_abort_claiming);631 632static void bd_end_claim(struct block_device *bdev, void *holder)633{634 struct block_device *whole = bdev_whole(bdev);635 bool unblock = false;636 637 /*638 * Release a claim on the device. The holder fields are protected with639 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking.640 */641 mutex_lock(&bdev_lock);642 WARN_ON_ONCE(bdev->bd_holder != holder);643 WARN_ON_ONCE(--bdev->bd_holders < 0);644 WARN_ON_ONCE(--whole->bd_holders < 0);645 if (!bdev->bd_holders) {646 mutex_lock(&bdev->bd_holder_lock);647 bdev->bd_holder = NULL;648 bdev->bd_holder_ops = NULL;649 mutex_unlock(&bdev->bd_holder_lock);650 if (bdev_test_flag(bdev, BD_WRITE_HOLDER))651 unblock = true;652 }653 if (!whole->bd_holders)654 whole->bd_holder = NULL;655 mutex_unlock(&bdev_lock);656 657 /*658 * If this was the last claim, remove holder link and unblock evpoll if659 * it was a write holder.660 */661 if (unblock) {662 disk_unblock_events(bdev->bd_disk);663 bdev_clear_flag(bdev, BD_WRITE_HOLDER);664 }665}666 667static void blkdev_flush_mapping(struct block_device *bdev)668{669 WARN_ON_ONCE(bdev->bd_holders);670 sync_blockdev(bdev);671 kill_bdev(bdev);672 bdev_write_inode(bdev);673}674 675static void blkdev_put_whole(struct block_device *bdev)676{677 if (atomic_dec_and_test(&bdev->bd_openers))678 blkdev_flush_mapping(bdev);679 if (bdev->bd_disk->fops->release)680 bdev->bd_disk->fops->release(bdev->bd_disk);681}682 683static int blkdev_get_whole(struct block_device *bdev, blk_mode_t mode)684{685 struct gendisk *disk = bdev->bd_disk;686 int ret;687 688 if (disk->fops->open) {689 ret = disk->fops->open(disk, mode);690 if (ret) {691 /* avoid ghost partitions on a removed medium */692 if (ret == -ENOMEDIUM &&693 test_bit(GD_NEED_PART_SCAN, &disk->state))694 bdev_disk_changed(disk, true);695 return ret;696 }697 }698 699 if (!atomic_read(&bdev->bd_openers))700 set_init_blocksize(bdev);701 atomic_inc(&bdev->bd_openers);702 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) {703 /*704 * Only return scanning errors if we are called from contexts705 * that explicitly want them, e.g. the BLKRRPART ioctl.706 */707 ret = bdev_disk_changed(disk, false);708 if (ret && (mode & BLK_OPEN_STRICT_SCAN)) {709 blkdev_put_whole(bdev);710 return ret;711 }712 }713 return 0;714}715 716static int blkdev_get_part(struct block_device *part, blk_mode_t mode)717{718 struct gendisk *disk = part->bd_disk;719 int ret;720 721 ret = blkdev_get_whole(bdev_whole(part), mode);722 if (ret)723 return ret;724 725 ret = -ENXIO;726 if (!bdev_nr_sectors(part))727 goto out_blkdev_put;728 729 if (!atomic_read(&part->bd_openers)) {730 disk->open_partitions++;731 set_init_blocksize(part);732 }733 atomic_inc(&part->bd_openers);734 return 0;735 736out_blkdev_put:737 blkdev_put_whole(bdev_whole(part));738 return ret;739}740 741int bdev_permission(dev_t dev, blk_mode_t mode, void *holder)742{743 int ret;744 745 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,746 MAJOR(dev), MINOR(dev),747 ((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) |748 ((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0));749 if (ret)750 return ret;751 752 /* Blocking writes requires exclusive opener */753 if (mode & BLK_OPEN_RESTRICT_WRITES && !holder)754 return -EINVAL;755 756 /*757 * We're using error pointers to indicate to ->release() when we758 * failed to open that block device. Also this doesn't make sense.759 */760 if (WARN_ON_ONCE(IS_ERR(holder)))761 return -EINVAL;762 763 return 0;764}765 766static void blkdev_put_part(struct block_device *part)767{768 struct block_device *whole = bdev_whole(part);769 770 if (atomic_dec_and_test(&part->bd_openers)) {771 blkdev_flush_mapping(part);772 whole->bd_disk->open_partitions--;773 }774 blkdev_put_whole(whole);775}776 777struct block_device *blkdev_get_no_open(dev_t dev)778{779 struct block_device *bdev;780 struct inode *inode;781 782 inode = ilookup(blockdev_superblock, dev);783 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {784 blk_request_module(dev);785 inode = ilookup(blockdev_superblock, dev);786 if (inode)787 pr_warn_ratelimited(788"block device autoloading is deprecated and will be removed.\n");789 }790 if (!inode)791 return NULL;792 793 /* switch from the inode reference to a device mode one: */794 bdev = &BDEV_I(inode)->bdev;795 if (!kobject_get_unless_zero(&bdev->bd_device.kobj))796 bdev = NULL;797 iput(inode);798 return bdev;799}800 801void blkdev_put_no_open(struct block_device *bdev)802{803 put_device(&bdev->bd_device);804}805 806static bool bdev_writes_blocked(struct block_device *bdev)807{808 return bdev->bd_writers < 0;809}810 811static void bdev_block_writes(struct block_device *bdev)812{813 bdev->bd_writers--;814}815 816static void bdev_unblock_writes(struct block_device *bdev)817{818 bdev->bd_writers++;819}820 821static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode)822{823 if (bdev_allow_write_mounted)824 return true;825 /* Writes blocked? */826 if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev))827 return false;828 if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0)829 return false;830 return true;831}832 833static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode)834{835 if (bdev_allow_write_mounted)836 return;837 838 /* Claim exclusive or shared write access. */839 if (mode & BLK_OPEN_RESTRICT_WRITES)840 bdev_block_writes(bdev);841 else if (mode & BLK_OPEN_WRITE)842 bdev->bd_writers++;843}844 845static inline bool bdev_unclaimed(const struct file *bdev_file)846{847 return bdev_file->private_data == BDEV_I(bdev_file->f_mapping->host);848}849 850static void bdev_yield_write_access(struct file *bdev_file)851{852 struct block_device *bdev;853 854 if (bdev_allow_write_mounted)855 return;856 857 if (bdev_unclaimed(bdev_file))858 return;859 860 bdev = file_bdev(bdev_file);861 862 if (bdev_file->f_mode & FMODE_WRITE_RESTRICTED)863 bdev_unblock_writes(bdev);864 else if (bdev_file->f_mode & FMODE_WRITE)865 bdev->bd_writers--;866}867 868/**869 * bdev_open - open a block device870 * @bdev: block device to open871 * @mode: open mode (BLK_OPEN_*)872 * @holder: exclusive holder identifier873 * @hops: holder operations874 * @bdev_file: file for the block device875 *876 * Open the block device. If @holder is not %NULL, the block device is opened877 * with exclusive access. Exclusive opens may nest for the same @holder.878 *879 * CONTEXT:880 * Might sleep.881 *882 * RETURNS:883 * zero on success, -errno on failure.884 */885int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,886 const struct blk_holder_ops *hops, struct file *bdev_file)887{888 bool unblock_events = true;889 struct gendisk *disk = bdev->bd_disk;890 int ret;891 892 if (holder) {893 mode |= BLK_OPEN_EXCL;894 ret = bd_prepare_to_claim(bdev, holder, hops);895 if (ret)896 return ret;897 } else {898 if (WARN_ON_ONCE(mode & BLK_OPEN_EXCL))899 return -EIO;900 }901 902 disk_block_events(disk);903 904 mutex_lock(&disk->open_mutex);905 ret = -ENXIO;906 if (!disk_live(disk))907 goto abort_claiming;908 if (!try_module_get(disk->fops->owner))909 goto abort_claiming;910 ret = -EBUSY;911 if (!bdev_may_open(bdev, mode))912 goto put_module;913 if (bdev_is_partition(bdev))914 ret = blkdev_get_part(bdev, mode);915 else916 ret = blkdev_get_whole(bdev, mode);917 if (ret)918 goto put_module;919 bdev_claim_write_access(bdev, mode);920 if (holder) {921 bd_finish_claiming(bdev, holder, hops);922 923 /*924 * Block event polling for write claims if requested. Any write925 * holder makes the write_holder state stick until all are926 * released. This is good enough and tracking individual927 * writeable reference is too fragile given the way @mode is928 * used in blkdev_get/put().929 */930 if ((mode & BLK_OPEN_WRITE) &&931 !bdev_test_flag(bdev, BD_WRITE_HOLDER) &&932 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) {933 bdev_set_flag(bdev, BD_WRITE_HOLDER);934 unblock_events = false;935 }936 }937 mutex_unlock(&disk->open_mutex);938 939 if (unblock_events)940 disk_unblock_events(disk);941 942 bdev_file->f_flags |= O_LARGEFILE;943 bdev_file->f_mode |= FMODE_CAN_ODIRECT;944 if (bdev_nowait(bdev))945 bdev_file->f_mode |= FMODE_NOWAIT;946 if (mode & BLK_OPEN_RESTRICT_WRITES)947 bdev_file->f_mode |= FMODE_WRITE_RESTRICTED;948 bdev_file->f_mapping = bdev->bd_mapping;949 bdev_file->f_wb_err = filemap_sample_wb_err(bdev_file->f_mapping);950 bdev_file->private_data = holder;951 952 return 0;953put_module:954 module_put(disk->fops->owner);955abort_claiming:956 if (holder)957 bd_abort_claiming(bdev, holder);958 mutex_unlock(&disk->open_mutex);959 disk_unblock_events(disk);960 return ret;961}962 963/*964 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk965 * associated with the floppy driver where it has allowed ioctls if the966 * file was opened for writing, but does not allow reads or writes.967 * Make sure that this quirk is reflected in @f_flags.968 *969 * It can also happen if a block device is opened as O_RDWR | O_WRONLY.970 */971static unsigned blk_to_file_flags(blk_mode_t mode)972{973 unsigned int flags = 0;974 975 if ((mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) ==976 (BLK_OPEN_READ | BLK_OPEN_WRITE))977 flags |= O_RDWR;978 else if (mode & BLK_OPEN_WRITE_IOCTL)979 flags |= O_RDWR | O_WRONLY;980 else if (mode & BLK_OPEN_WRITE)981 flags |= O_WRONLY;982 else if (mode & BLK_OPEN_READ)983 flags |= O_RDONLY; /* homeopathic, because O_RDONLY is 0 */984 else985 WARN_ON_ONCE(true);986 987 if (mode & BLK_OPEN_NDELAY)988 flags |= O_NDELAY;989 990 return flags;991}992 993struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,994 const struct blk_holder_ops *hops)995{996 struct file *bdev_file;997 struct block_device *bdev;998 unsigned int flags;999 int ret;1000 1001 ret = bdev_permission(dev, mode, holder);1002 if (ret)1003 return ERR_PTR(ret);1004 1005 bdev = blkdev_get_no_open(dev);1006 if (!bdev)1007 return ERR_PTR(-ENXIO);1008 1009 flags = blk_to_file_flags(mode);1010 bdev_file = alloc_file_pseudo_noaccount(BD_INODE(bdev),1011 blockdev_mnt, "", flags | O_LARGEFILE, &def_blk_fops);1012 if (IS_ERR(bdev_file)) {1013 blkdev_put_no_open(bdev);1014 return bdev_file;1015 }1016 ihold(BD_INODE(bdev));1017 1018 ret = bdev_open(bdev, mode, holder, hops, bdev_file);1019 if (ret) {1020 /* We failed to open the block device. Let ->release() know. */1021 bdev_file->private_data = ERR_PTR(ret);1022 fput(bdev_file);1023 return ERR_PTR(ret);1024 }1025 return bdev_file;1026}1027EXPORT_SYMBOL(bdev_file_open_by_dev);1028 1029struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,1030 void *holder,1031 const struct blk_holder_ops *hops)1032{1033 struct file *file;1034 dev_t dev;1035 int error;1036 1037 error = lookup_bdev(path, &dev);1038 if (error)1039 return ERR_PTR(error);1040 1041 file = bdev_file_open_by_dev(dev, mode, holder, hops);1042 if (!IS_ERR(file) && (mode & BLK_OPEN_WRITE)) {1043 if (bdev_read_only(file_bdev(file))) {1044 fput(file);1045 file = ERR_PTR(-EACCES);1046 }1047 }1048 1049 return file;1050}1051EXPORT_SYMBOL(bdev_file_open_by_path);1052 1053static inline void bd_yield_claim(struct file *bdev_file)1054{1055 struct block_device *bdev = file_bdev(bdev_file);1056 void *holder = bdev_file->private_data;1057 1058 lockdep_assert_held(&bdev->bd_disk->open_mutex);1059 1060 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder)))1061 return;1062 1063 if (!bdev_unclaimed(bdev_file))1064 bd_end_claim(bdev, holder);1065}1066 1067void bdev_release(struct file *bdev_file)1068{1069 struct block_device *bdev = file_bdev(bdev_file);1070 void *holder = bdev_file->private_data;1071 struct gendisk *disk = bdev->bd_disk;1072 1073 /* We failed to open that block device. */1074 if (IS_ERR(holder))1075 goto put_no_open;1076 1077 /*1078 * Sync early if it looks like we're the last one. If someone else1079 * opens the block device between now and the decrement of bd_openers1080 * then we did a sync that we didn't need to, but that's not the end1081 * of the world and we want to avoid long (could be several minute)1082 * syncs while holding the mutex.1083 */1084 if (atomic_read(&bdev->bd_openers) == 1)1085 sync_blockdev(bdev);1086 1087 mutex_lock(&disk->open_mutex);1088 bdev_yield_write_access(bdev_file);1089 1090 if (holder)1091 bd_yield_claim(bdev_file);1092 1093 /*1094 * Trigger event checking and tell drivers to flush MEDIA_CHANGE1095 * event. This is to ensure detection of media removal commanded1096 * from userland - e.g. eject(1).1097 */1098 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE);1099 1100 if (bdev_is_partition(bdev))1101 blkdev_put_part(bdev);1102 else1103 blkdev_put_whole(bdev);1104 mutex_unlock(&disk->open_mutex);1105 1106 module_put(disk->fops->owner);1107put_no_open:1108 blkdev_put_no_open(bdev);1109}1110 1111/**1112 * bdev_fput - yield claim to the block device and put the file1113 * @bdev_file: open block device1114 *1115 * Yield claim on the block device and put the file. Ensure that the1116 * block device can be reclaimed before the file is closed which is a1117 * deferred operation.1118 */1119void bdev_fput(struct file *bdev_file)1120{1121 if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops))1122 return;1123 1124 if (bdev_file->private_data) {1125 struct block_device *bdev = file_bdev(bdev_file);1126 struct gendisk *disk = bdev->bd_disk;1127 1128 mutex_lock(&disk->open_mutex);1129 bdev_yield_write_access(bdev_file);1130 bd_yield_claim(bdev_file);1131 /*1132 * Tell release we already gave up our hold on the1133 * device and if write restrictions are available that1134 * we already gave up write access to the device.1135 */1136 bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host);1137 mutex_unlock(&disk->open_mutex);1138 }1139 1140 fput(bdev_file);1141}1142EXPORT_SYMBOL(bdev_fput);1143 1144/**1145 * lookup_bdev() - Look up a struct block_device by name.1146 * @pathname: Name of the block device in the filesystem.1147 * @dev: Pointer to the block device's dev_t, if found.1148 *1149 * Lookup the block device's dev_t at @pathname in the current1150 * namespace if possible and return it in @dev.1151 *1152 * Context: May sleep.1153 * Return: 0 if succeeded, negative errno otherwise.1154 */1155int lookup_bdev(const char *pathname, dev_t *dev)1156{1157 struct inode *inode;1158 struct path path;1159 int error;1160 1161 if (!pathname || !*pathname)1162 return -EINVAL;1163 1164 error = kern_path(pathname, LOOKUP_FOLLOW, &path);1165 if (error)1166 return error;1167 1168 inode = d_backing_inode(path.dentry);1169 error = -ENOTBLK;1170 if (!S_ISBLK(inode->i_mode))1171 goto out_path_put;1172 error = -EACCES;1173 if (!may_open_dev(&path))1174 goto out_path_put;1175 1176 *dev = inode->i_rdev;1177 error = 0;1178out_path_put:1179 path_put(&path);1180 return error;1181}1182EXPORT_SYMBOL(lookup_bdev);1183 1184/**1185 * bdev_mark_dead - mark a block device as dead1186 * @bdev: block device to operate on1187 * @surprise: indicate a surprise removal1188 *1189 * Tell the file system that this devices or media is dead. If @surprise is set1190 * to %true the device or media is already gone, if not we are preparing for an1191 * orderly removal.1192 *1193 * This calls into the file system, which then typicall syncs out all dirty data1194 * and writes back inodes and then invalidates any cached data in the inodes on1195 * the file system. In addition we also invalidate the block device mapping.1196 */1197void bdev_mark_dead(struct block_device *bdev, bool surprise)1198{1199 mutex_lock(&bdev->bd_holder_lock);1200 if (bdev->bd_holder_ops && bdev->bd_holder_ops->mark_dead)1201 bdev->bd_holder_ops->mark_dead(bdev, surprise);1202 else {1203 mutex_unlock(&bdev->bd_holder_lock);1204 sync_blockdev(bdev);1205 }1206 1207 invalidate_bdev(bdev);1208}1209/*1210 * New drivers should not use this directly. There are some drivers however1211 * that needs this for historical reasons. For example, the DASD driver has1212 * historically had a shutdown to offline mode that doesn't actually remove the1213 * gendisk that otherwise looks a lot like a safe device removal.1214 */1215EXPORT_SYMBOL_GPL(bdev_mark_dead);1216 1217void sync_bdevs(bool wait)1218{1219 struct inode *inode, *old_inode = NULL;1220 1221 spin_lock(&blockdev_superblock->s_inode_list_lock);1222 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {1223 struct address_space *mapping = inode->i_mapping;1224 struct block_device *bdev;1225 1226 spin_lock(&inode->i_lock);1227 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||1228 mapping->nrpages == 0) {1229 spin_unlock(&inode->i_lock);1230 continue;1231 }1232 __iget(inode);1233 spin_unlock(&inode->i_lock);1234 spin_unlock(&blockdev_superblock->s_inode_list_lock);1235 /*1236 * We hold a reference to 'inode' so it couldn't have been1237 * removed from s_inodes list while we dropped the1238 * s_inode_list_lock We cannot iput the inode now as we can1239 * be holding the last reference and we cannot iput it under1240 * s_inode_list_lock. So we keep the reference and iput it1241 * later.1242 */1243 iput(old_inode);1244 old_inode = inode;1245 bdev = I_BDEV(inode);1246 1247 mutex_lock(&bdev->bd_disk->open_mutex);1248 if (!atomic_read(&bdev->bd_openers)) {1249 ; /* skip */1250 } else if (wait) {1251 /*1252 * We keep the error status of individual mapping so1253 * that applications can catch the writeback error using1254 * fsync(2). See filemap_fdatawait_keep_errors() for1255 * details.1256 */1257 filemap_fdatawait_keep_errors(inode->i_mapping);1258 } else {1259 filemap_fdatawrite(inode->i_mapping);1260 }1261 mutex_unlock(&bdev->bd_disk->open_mutex);1262 1263 spin_lock(&blockdev_superblock->s_inode_list_lock);1264 }1265 spin_unlock(&blockdev_superblock->s_inode_list_lock);1266 iput(old_inode);1267}1268 1269/*1270 * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.1271 */1272void bdev_statx(struct path *path, struct kstat *stat,1273 u32 request_mask)1274{1275 struct inode *backing_inode;1276 struct block_device *bdev;1277 1278 if (!(request_mask & (STATX_DIOALIGN | STATX_WRITE_ATOMIC)))1279 return;1280 1281 backing_inode = d_backing_inode(path->dentry);1282 1283 /*1284 * Note that backing_inode is the inode of a block device node file,1285 * not the block device's internal inode. Therefore it is *not* valid1286 * to use I_BDEV() here; the block device has to be looked up by i_rdev1287 * instead.1288 */1289 bdev = blkdev_get_no_open(backing_inode->i_rdev);1290 if (!bdev)1291 return;1292 1293 if (request_mask & STATX_DIOALIGN) {1294 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;1295 stat->dio_offset_align = bdev_logical_block_size(bdev);1296 stat->result_mask |= STATX_DIOALIGN;1297 }1298 1299 if (request_mask & STATX_WRITE_ATOMIC && bdev_can_atomic_write(bdev)) {1300 struct request_queue *bd_queue = bdev->bd_queue;1301 1302 generic_fill_statx_atomic_writes(stat,1303 queue_atomic_write_unit_min_bytes(bd_queue),1304 queue_atomic_write_unit_max_bytes(bd_queue));1305 }1306 1307 blkdev_put_no_open(bdev);1308}1309 1310bool disk_live(struct gendisk *disk)1311{1312 return !inode_unhashed(BD_INODE(disk->part0));1313}1314EXPORT_SYMBOL_GPL(disk_live);1315 1316unsigned int block_size(struct block_device *bdev)1317{1318 return 1 << BD_INODE(bdev)->i_blkbits;1319}1320EXPORT_SYMBOL_GPL(block_size);1321 1322static int __init setup_bdev_allow_write_mounted(char *str)1323{1324 if (kstrtobool(str, &bdev_allow_write_mounted))1325 pr_warn("Invalid option string for bdev_allow_write_mounted:"1326 " '%s'\n", str);1327 return 1;1328}1329__setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted);1330