371 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Functions related to generic helpers functions4 */5#include <linux/kernel.h>6#include <linux/module.h>7#include <linux/bio.h>8#include <linux/blkdev.h>9#include <linux/scatterlist.h>10 11#include "blk.h"12 13static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector)14{15 unsigned int discard_granularity = bdev_discard_granularity(bdev);16 sector_t granularity_aligned_sector;17 18 if (bdev_is_partition(bdev))19 sector += bdev->bd_start_sect;20 21 granularity_aligned_sector =22 round_up(sector, discard_granularity >> SECTOR_SHIFT);23 24 /*25 * Make sure subsequent bios start aligned to the discard granularity if26 * it needs to be split.27 */28 if (granularity_aligned_sector != sector)29 return granularity_aligned_sector - sector;30 31 /*32 * Align the bio size to the discard granularity to make splitting the bio33 * at discard granularity boundaries easier in the driver if needed.34 */35 return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT;36}37 38struct bio *blk_alloc_discard_bio(struct block_device *bdev,39 sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask)40{41 sector_t bio_sects = min(*nr_sects, bio_discard_limit(bdev, *sector));42 struct bio *bio;43 44 if (!bio_sects)45 return NULL;46 47 bio = bio_alloc(bdev, 0, REQ_OP_DISCARD, gfp_mask);48 if (!bio)49 return NULL;50 bio->bi_iter.bi_sector = *sector;51 bio->bi_iter.bi_size = bio_sects << SECTOR_SHIFT;52 *sector += bio_sects;53 *nr_sects -= bio_sects;54 /*55 * We can loop for a long time in here if someone does full device56 * discards (like mkfs). Be nice and allow us to schedule out to avoid57 * softlocking if preempt is disabled.58 */59 cond_resched();60 return bio;61}62 63int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,64 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop)65{66 struct bio *bio;67 68 while ((bio = blk_alloc_discard_bio(bdev, §or, &nr_sects,69 gfp_mask)))70 *biop = bio_chain_and_submit(*biop, bio);71 return 0;72}73EXPORT_SYMBOL(__blkdev_issue_discard);74 75/**76 * blkdev_issue_discard - queue a discard77 * @bdev: blockdev to issue discard for78 * @sector: start sector79 * @nr_sects: number of sectors to discard80 * @gfp_mask: memory allocation flags (for bio_alloc)81 *82 * Description:83 * Issue a discard request for the sectors in question.84 */85int blkdev_issue_discard(struct block_device *bdev, sector_t sector,86 sector_t nr_sects, gfp_t gfp_mask)87{88 struct bio *bio = NULL;89 struct blk_plug plug;90 int ret;91 92 blk_start_plug(&plug);93 ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio);94 if (!ret && bio) {95 ret = submit_bio_wait(bio);96 if (ret == -EOPNOTSUPP)97 ret = 0;98 bio_put(bio);99 }100 blk_finish_plug(&plug);101 102 return ret;103}104EXPORT_SYMBOL(blkdev_issue_discard);105 106static sector_t bio_write_zeroes_limit(struct block_device *bdev)107{108 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;109 110 return min(bdev_write_zeroes_sectors(bdev),111 (UINT_MAX >> SECTOR_SHIFT) & ~bs_mask);112}113 114/*115 * There is no reliable way for the SCSI subsystem to determine whether a116 * device supports a WRITE SAME operation without actually performing a write117 * to media. As a result, write_zeroes is enabled by default and will be118 * disabled if a zeroing operation subsequently fails. This means that this119 * queue limit is likely to change at runtime.120 */121static void __blkdev_issue_write_zeroes(struct block_device *bdev,122 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,123 struct bio **biop, unsigned flags, sector_t limit)124{125 126 while (nr_sects) {127 unsigned int len = min(nr_sects, limit);128 struct bio *bio;129 130 if ((flags & BLKDEV_ZERO_KILLABLE) &&131 fatal_signal_pending(current))132 break;133 134 bio = bio_alloc(bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);135 bio->bi_iter.bi_sector = sector;136 if (flags & BLKDEV_ZERO_NOUNMAP)137 bio->bi_opf |= REQ_NOUNMAP;138 139 bio->bi_iter.bi_size = len << SECTOR_SHIFT;140 *biop = bio_chain_and_submit(*biop, bio);141 142 nr_sects -= len;143 sector += len;144 cond_resched();145 }146}147 148static int blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector,149 sector_t nr_sects, gfp_t gfp, unsigned flags)150{151 sector_t limit = bio_write_zeroes_limit(bdev);152 struct bio *bio = NULL;153 struct blk_plug plug;154 int ret = 0;155 156 blk_start_plug(&plug);157 __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp, &bio,158 flags, limit);159 if (bio) {160 if ((flags & BLKDEV_ZERO_KILLABLE) &&161 fatal_signal_pending(current)) {162 bio_await_chain(bio);163 blk_finish_plug(&plug);164 return -EINTR;165 }166 ret = submit_bio_wait(bio);167 bio_put(bio);168 }169 blk_finish_plug(&plug);170 171 /*172 * For some devices there is no non-destructive way to verify whether173 * WRITE ZEROES is actually supported. These will clear the capability174 * on an I/O error, in which case we'll turn any error into175 * "not supported" here.176 */177 if (ret && !bdev_write_zeroes_sectors(bdev))178 return -EOPNOTSUPP;179 return ret;180}181 182/*183 * Convert a number of 512B sectors to a number of pages.184 * The result is limited to a number of pages that can fit into a BIO.185 * Also make sure that the result is always at least 1 (page) for the cases186 * where nr_sects is lower than the number of sectors in a page.187 */188static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)189{190 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);191 192 return min(pages, (sector_t)BIO_MAX_VECS);193}194 195static void __blkdev_issue_zero_pages(struct block_device *bdev,196 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,197 struct bio **biop, unsigned int flags)198{199 while (nr_sects) {200 unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects);201 struct bio *bio;202 203 bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask);204 bio->bi_iter.bi_sector = sector;205 206 if ((flags & BLKDEV_ZERO_KILLABLE) &&207 fatal_signal_pending(current))208 break;209 210 do {211 unsigned int len, added;212 213 len = min_t(sector_t,214 PAGE_SIZE, nr_sects << SECTOR_SHIFT);215 added = bio_add_page(bio, ZERO_PAGE(0), len, 0);216 if (added < len)217 break;218 nr_sects -= added >> SECTOR_SHIFT;219 sector += added >> SECTOR_SHIFT;220 } while (nr_sects);221 222 *biop = bio_chain_and_submit(*biop, bio);223 cond_resched();224 }225}226 227static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector,228 sector_t nr_sects, gfp_t gfp, unsigned flags)229{230 struct bio *bio = NULL;231 struct blk_plug plug;232 int ret = 0;233 234 if (flags & BLKDEV_ZERO_NOFALLBACK)235 return -EOPNOTSUPP;236 237 blk_start_plug(&plug);238 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp, &bio, flags);239 if (bio) {240 if ((flags & BLKDEV_ZERO_KILLABLE) &&241 fatal_signal_pending(current)) {242 bio_await_chain(bio);243 blk_finish_plug(&plug);244 return -EINTR;245 }246 ret = submit_bio_wait(bio);247 bio_put(bio);248 }249 blk_finish_plug(&plug);250 251 return ret;252}253 254/**255 * __blkdev_issue_zeroout - generate number of zero filed write bios256 * @bdev: blockdev to issue257 * @sector: start sector258 * @nr_sects: number of sectors to write259 * @gfp_mask: memory allocation flags (for bio_alloc)260 * @biop: pointer to anchor bio261 * @flags: controls detailed behavior262 *263 * Description:264 * Zero-fill a block range, either using hardware offload or by explicitly265 * writing zeroes to the device.266 *267 * If a device is using logical block provisioning, the underlying space will268 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP.269 *270 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return271 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.272 */273int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,274 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,275 unsigned flags)276{277 sector_t limit = bio_write_zeroes_limit(bdev);278 279 if (bdev_read_only(bdev))280 return -EPERM;281 282 if (limit) {283 __blkdev_issue_write_zeroes(bdev, sector, nr_sects,284 gfp_mask, biop, flags, limit);285 } else {286 if (flags & BLKDEV_ZERO_NOFALLBACK)287 return -EOPNOTSUPP;288 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,289 biop, flags);290 }291 return 0;292}293EXPORT_SYMBOL(__blkdev_issue_zeroout);294 295/**296 * blkdev_issue_zeroout - zero-fill a block range297 * @bdev: blockdev to write298 * @sector: start sector299 * @nr_sects: number of sectors to write300 * @gfp_mask: memory allocation flags (for bio_alloc)301 * @flags: controls detailed behavior302 *303 * Description:304 * Zero-fill a block range, either using hardware offload or by explicitly305 * writing zeroes to the device. See __blkdev_issue_zeroout() for the306 * valid values for %flags.307 */308int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,309 sector_t nr_sects, gfp_t gfp_mask, unsigned flags)310{311 int ret;312 313 if ((sector | nr_sects) & ((bdev_logical_block_size(bdev) >> 9) - 1))314 return -EINVAL;315 if (bdev_read_only(bdev))316 return -EPERM;317 318 if (bdev_write_zeroes_sectors(bdev)) {319 ret = blkdev_issue_write_zeroes(bdev, sector, nr_sects,320 gfp_mask, flags);321 if (ret != -EOPNOTSUPP)322 return ret;323 }324 325 return blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, flags);326}327EXPORT_SYMBOL(blkdev_issue_zeroout);328 329int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,330 sector_t nr_sects, gfp_t gfp)331{332 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;333 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev);334 struct bio *bio = NULL;335 struct blk_plug plug;336 int ret = 0;337 338 /* make sure that "len << SECTOR_SHIFT" doesn't overflow */339 if (max_sectors > UINT_MAX >> SECTOR_SHIFT)340 max_sectors = UINT_MAX >> SECTOR_SHIFT;341 max_sectors &= ~bs_mask;342 343 if (max_sectors == 0)344 return -EOPNOTSUPP;345 if ((sector | nr_sects) & bs_mask)346 return -EINVAL;347 if (bdev_read_only(bdev))348 return -EPERM;349 350 blk_start_plug(&plug);351 while (nr_sects) {352 unsigned int len = min_t(sector_t, nr_sects, max_sectors);353 354 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp);355 bio->bi_iter.bi_sector = sector;356 bio->bi_iter.bi_size = len << SECTOR_SHIFT;357 358 sector += len;359 nr_sects -= len;360 cond_resched();361 }362 if (bio) {363 ret = submit_bio_wait(bio);364 bio_put(bio);365 }366 blk_finish_plug(&plug);367 368 return ret;369}370EXPORT_SYMBOL(blkdev_issue_secure_erase);371