469 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.4 */5 6#include <linux/blkdev.h>7#include <linux/mm.h>8#include <linux/sched/mm.h>9#include <linux/slab.h>10#include <linux/bitmap.h>11 12#include "dm-core.h"13 14#define DM_MSG_PREFIX "zone"15 16/*17 * For internal zone reports bypassing the top BIO submission path.18 */19static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,20 sector_t sector, unsigned int nr_zones,21 report_zones_cb cb, void *data)22{23 struct gendisk *disk = md->disk;24 int ret;25 struct dm_report_zones_args args = {26 .next_sector = sector,27 .orig_data = data,28 .orig_cb = cb,29 };30 31 do {32 struct dm_target *tgt;33 34 tgt = dm_table_find_target(t, args.next_sector);35 if (WARN_ON_ONCE(!tgt->type->report_zones))36 return -EIO;37 38 args.tgt = tgt;39 ret = tgt->type->report_zones(tgt, &args,40 nr_zones - args.zone_idx);41 if (ret < 0)42 return ret;43 } while (args.zone_idx < nr_zones &&44 args.next_sector < get_capacity(disk));45 46 return args.zone_idx;47}48 49/*50 * User facing dm device block device report zone operation. This calls the51 * report_zones operation for each target of a device table. This operation is52 * generally implemented by targets using dm_report_zones().53 */54int dm_blk_report_zones(struct gendisk *disk, sector_t sector,55 unsigned int nr_zones, report_zones_cb cb, void *data)56{57 struct mapped_device *md = disk->private_data;58 struct dm_table *map;59 int srcu_idx, ret;60 61 if (!md->zone_revalidate_map) {62 /* Regular user context */63 if (dm_suspended_md(md))64 return -EAGAIN;65 66 map = dm_get_live_table(md, &srcu_idx);67 if (!map)68 return -EIO;69 } else {70 /* Zone revalidation during __bind() */71 map = md->zone_revalidate_map;72 }73 74 ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);75 76 if (!md->zone_revalidate_map)77 dm_put_live_table(md, srcu_idx);78 79 return ret;80}81 82static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,83 void *data)84{85 struct dm_report_zones_args *args = data;86 sector_t sector_diff = args->tgt->begin - args->start;87 88 /*89 * Ignore zones beyond the target range.90 */91 if (zone->start >= args->start + args->tgt->len)92 return 0;93 94 /*95 * Remap the start sector and write pointer position of the zone96 * to match its position in the target range.97 */98 zone->start += sector_diff;99 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {100 if (zone->cond == BLK_ZONE_COND_FULL)101 zone->wp = zone->start + zone->len;102 else if (zone->cond == BLK_ZONE_COND_EMPTY)103 zone->wp = zone->start;104 else105 zone->wp += sector_diff;106 }107 108 args->next_sector = zone->start + zone->len;109 return args->orig_cb(zone, args->zone_idx++, args->orig_data);110}111 112/*113 * Helper for drivers of zoned targets to implement struct target_type114 * report_zones operation.115 */116int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,117 struct dm_report_zones_args *args, unsigned int nr_zones)118{119 /*120 * Set the target mapping start sector first so that121 * dm_report_zones_cb() can correctly remap zone information.122 */123 args->start = start;124 125 return blkdev_report_zones(bdev, sector, nr_zones,126 dm_report_zones_cb, args);127}128EXPORT_SYMBOL_GPL(dm_report_zones);129 130bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)131{132 struct request_queue *q = md->queue;133 134 if (!blk_queue_is_zoned(q))135 return false;136 137 switch (bio_op(bio)) {138 case REQ_OP_WRITE_ZEROES:139 case REQ_OP_WRITE:140 return !op_is_flush(bio->bi_opf) && bio_sectors(bio);141 default:142 return false;143 }144}145 146/*147 * Revalidate the zones of a mapped device to initialize resource necessary148 * for zone append emulation. Note that we cannot simply use the block layer149 * blk_revalidate_disk_zones() function here as the mapped device is suspended150 * (this is called from __bind() context).151 */152int dm_revalidate_zones(struct dm_table *t, struct request_queue *q)153{154 struct mapped_device *md = t->md;155 struct gendisk *disk = md->disk;156 int ret;157 158 if (!get_capacity(disk))159 return 0;160 161 /* Revalidate only if something changed. */162 if (!disk->nr_zones || disk->nr_zones != md->nr_zones) {163 DMINFO("%s using %s zone append",164 disk->disk_name,165 queue_emulates_zone_append(q) ? "emulated" : "native");166 md->nr_zones = 0;167 }168 169 if (md->nr_zones)170 return 0;171 172 /*173 * Our table is not live yet. So the call to dm_get_live_table()174 * in dm_blk_report_zones() will fail. Set a temporary pointer to175 * our table for dm_blk_report_zones() to use directly.176 */177 md->zone_revalidate_map = t;178 ret = blk_revalidate_disk_zones(disk);179 md->zone_revalidate_map = NULL;180 181 if (ret) {182 DMERR("Revalidate zones failed %d", ret);183 return ret;184 }185 186 md->nr_zones = disk->nr_zones;187 188 return 0;189}190 191static int device_not_zone_append_capable(struct dm_target *ti,192 struct dm_dev *dev, sector_t start,193 sector_t len, void *data)194{195 return !bdev_is_zoned(dev->bdev);196}197 198static bool dm_table_supports_zone_append(struct dm_table *t)199{200 for (unsigned int i = 0; i < t->num_targets; i++) {201 struct dm_target *ti = dm_table_get_target(t, i);202 203 if (ti->emulate_zone_append)204 return false;205 206 if (!ti->type->iterate_devices ||207 ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))208 return false;209 }210 211 return true;212}213 214struct dm_device_zone_count {215 sector_t start;216 sector_t len;217 unsigned int total_nr_seq_zones;218 unsigned int target_nr_seq_zones;219};220 221/*222 * Count the total number of and the number of mapped sequential zones of a223 * target zoned device.224 */225static int dm_device_count_zones_cb(struct blk_zone *zone,226 unsigned int idx, void *data)227{228 struct dm_device_zone_count *zc = data;229 230 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {231 zc->total_nr_seq_zones++;232 if (zone->start >= zc->start &&233 zone->start < zc->start + zc->len)234 zc->target_nr_seq_zones++;235 }236 237 return 0;238}239 240static int dm_device_count_zones(struct dm_dev *dev,241 struct dm_device_zone_count *zc)242{243 int ret;244 245 ret = blkdev_report_zones(dev->bdev, 0, BLK_ALL_ZONES,246 dm_device_count_zones_cb, zc);247 if (ret < 0)248 return ret;249 if (!ret)250 return -EIO;251 return 0;252}253 254struct dm_zone_resource_limits {255 unsigned int mapped_nr_seq_zones;256 struct queue_limits *lim;257 bool reliable_limits;258};259 260static int device_get_zone_resource_limits(struct dm_target *ti,261 struct dm_dev *dev, sector_t start,262 sector_t len, void *data)263{264 struct dm_zone_resource_limits *zlim = data;265 struct gendisk *disk = dev->bdev->bd_disk;266 unsigned int max_open_zones, max_active_zones;267 int ret;268 struct dm_device_zone_count zc = {269 .start = start,270 .len = len,271 };272 273 /*274 * If the target is not the whole device, the device zone resources may275 * be shared between different targets. Check this by counting the276 * number of mapped sequential zones: if this number is smaller than the277 * total number of sequential zones of the target device, then resource278 * sharing may happen and the zone limits will not be reliable.279 */280 ret = dm_device_count_zones(dev, &zc);281 if (ret) {282 DMERR("Count %s zones failed %d", disk->disk_name, ret);283 return ret;284 }285 286 /*287 * If the target does not map any sequential zones, then we do not need288 * any zone resource limits.289 */290 if (!zc.target_nr_seq_zones)291 return 0;292 293 /*294 * If the target does not map all sequential zones, the limits295 * will not be reliable and we cannot use REQ_OP_ZONE_RESET_ALL.296 */297 if (zc.target_nr_seq_zones < zc.total_nr_seq_zones) {298 zlim->reliable_limits = false;299 ti->zone_reset_all_supported = false;300 }301 302 /*303 * If the target maps less sequential zones than the limit values, then304 * we do not have limits for this target.305 */306 max_active_zones = disk->queue->limits.max_active_zones;307 if (max_active_zones >= zc.target_nr_seq_zones)308 max_active_zones = 0;309 zlim->lim->max_active_zones =310 min_not_zero(max_active_zones, zlim->lim->max_active_zones);311 312 max_open_zones = disk->queue->limits.max_open_zones;313 if (max_open_zones >= zc.target_nr_seq_zones)314 max_open_zones = 0;315 zlim->lim->max_open_zones =316 min_not_zero(max_open_zones, zlim->lim->max_open_zones);317 318 /*319 * Also count the total number of sequential zones for the mapped320 * device so that when we are done inspecting all its targets, we are321 * able to check if the mapped device actually has any sequential zones.322 */323 zlim->mapped_nr_seq_zones += zc.target_nr_seq_zones;324 325 return 0;326}327 328int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q,329 struct queue_limits *lim)330{331 struct mapped_device *md = t->md;332 struct gendisk *disk = md->disk;333 struct dm_zone_resource_limits zlim = {334 .reliable_limits = true,335 .lim = lim,336 };337 338 /*339 * Check if zone append is natively supported, and if not, set the340 * mapped device queue as needing zone append emulation.341 */342 WARN_ON_ONCE(queue_is_mq(q));343 if (dm_table_supports_zone_append(t)) {344 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);345 } else {346 set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);347 lim->max_zone_append_sectors = 0;348 }349 350 /*351 * Determine the max open and max active zone limits for the mapped352 * device by inspecting the zone resource limits and the zones mapped353 * by each target.354 */355 for (unsigned int i = 0; i < t->num_targets; i++) {356 struct dm_target *ti = dm_table_get_target(t, i);357 358 /*359 * Assume that the target can accept REQ_OP_ZONE_RESET_ALL.360 * device_get_zone_resource_limits() may adjust this if one of361 * the device used by the target does not have all its362 * sequential write required zones mapped.363 */364 ti->zone_reset_all_supported = true;365 366 if (!ti->type->iterate_devices ||367 ti->type->iterate_devices(ti,368 device_get_zone_resource_limits, &zlim)) {369 DMERR("Could not determine %s zone resource limits",370 disk->disk_name);371 return -ENODEV;372 }373 }374 375 /*376 * If we only have conventional zones mapped, expose the mapped device377 + as a regular device.378 */379 if (!zlim.mapped_nr_seq_zones) {380 lim->max_open_zones = 0;381 lim->max_active_zones = 0;382 lim->max_zone_append_sectors = 0;383 lim->zone_write_granularity = 0;384 lim->chunk_sectors = 0;385 lim->features &= ~BLK_FEAT_ZONED;386 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);387 md->nr_zones = 0;388 disk->nr_zones = 0;389 return 0;390 }391 392 /*393 * Warn once (when the capacity is not yet set) if the mapped device is394 * partially using zone resources of the target devices as that leads to395 * unreliable limits, i.e. if another mapped device uses the same396 * underlying devices, we cannot enforce zone limits to guarantee that397 * writing will not lead to errors. Note that we really should return398 * an error for such case but there is no easy way to find out if399 * another mapped device uses the same underlying zoned devices.400 */401 if (!get_capacity(disk) && !zlim.reliable_limits)402 DMWARN("%s zone resource limits may be unreliable",403 disk->disk_name);404 405 if (lim->features & BLK_FEAT_ZONED &&406 !static_key_enabled(&zoned_enabled.key))407 static_branch_enable(&zoned_enabled);408 return 0;409}410 411/*412 * IO completion callback called from clone_endio().413 */414void dm_zone_endio(struct dm_io *io, struct bio *clone)415{416 struct mapped_device *md = io->md;417 struct gendisk *disk = md->disk;418 struct bio *orig_bio = io->orig_bio;419 420 /*421 * Get the offset within the zone of the written sector422 * and add that to the original bio sector position.423 */424 if (clone->bi_status == BLK_STS_OK &&425 bio_op(clone) == REQ_OP_ZONE_APPEND) {426 sector_t mask = bdev_zone_sectors(disk->part0) - 1;427 428 orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask;429 }430 431 return;432}433 434static int dm_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx,435 void *data)436{437 /*438 * For an all-zones reset, ignore conventional, empty, read-only439 * and offline zones.440 */441 switch (zone->cond) {442 case BLK_ZONE_COND_NOT_WP:443 case BLK_ZONE_COND_EMPTY:444 case BLK_ZONE_COND_READONLY:445 case BLK_ZONE_COND_OFFLINE:446 return 0;447 default:448 set_bit(idx, (unsigned long *)data);449 return 0;450 }451}452 453int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t,454 sector_t sector, unsigned int nr_zones,455 unsigned long *need_reset)456{457 int ret;458 459 ret = dm_blk_do_report_zones(md, t, sector, nr_zones,460 dm_zone_need_reset_cb, need_reset);461 if (ret != nr_zones) {462 DMERR("Get %s zone reset bitmap failed\n",463 md->disk->disk_name);464 return -EIO;465 }466 467 return 0;468}469