865 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Functions related to sysfs handling4 */5#include <linux/kernel.h>6#include <linux/slab.h>7#include <linux/module.h>8#include <linux/bio.h>9#include <linux/blkdev.h>10#include <linux/backing-dev.h>11#include <linux/blktrace_api.h>12#include <linux/debugfs.h>13 14#include "blk.h"15#include "blk-mq.h"16#include "blk-mq-debugfs.h"17#include "blk-mq-sched.h"18#include "blk-rq-qos.h"19#include "blk-wbt.h"20#include "blk-cgroup.h"21#include "blk-throttle.h"22 23struct queue_sysfs_entry {24 struct attribute attr;25 ssize_t (*show)(struct gendisk *disk, char *page);26 int (*load_module)(struct gendisk *disk, const char *page, size_t count);27 ssize_t (*store)(struct gendisk *disk, const char *page, size_t count);28};29 30static ssize_t31queue_var_show(unsigned long var, char *page)32{33 return sprintf(page, "%lu\n", var);34}35 36static ssize_t37queue_var_store(unsigned long *var, const char *page, size_t count)38{39 int err;40 unsigned long v;41 42 err = kstrtoul(page, 10, &v);43 if (err || v > UINT_MAX)44 return -EINVAL;45 46 *var = v;47 48 return count;49}50 51static ssize_t queue_requests_show(struct gendisk *disk, char *page)52{53 return queue_var_show(disk->queue->nr_requests, page);54}55 56static ssize_t57queue_requests_store(struct gendisk *disk, const char *page, size_t count)58{59 unsigned long nr;60 int ret, err;61 62 if (!queue_is_mq(disk->queue))63 return -EINVAL;64 65 ret = queue_var_store(&nr, page, count);66 if (ret < 0)67 return ret;68 69 if (nr < BLKDEV_MIN_RQ)70 nr = BLKDEV_MIN_RQ;71 72 err = blk_mq_update_nr_requests(disk->queue, nr);73 if (err)74 return err;75 76 return ret;77}78 79static ssize_t queue_ra_show(struct gendisk *disk, char *page)80{81 return queue_var_show(disk->bdi->ra_pages << (PAGE_SHIFT - 10), page);82}83 84static ssize_t85queue_ra_store(struct gendisk *disk, const char *page, size_t count)86{87 unsigned long ra_kb;88 ssize_t ret;89 90 ret = queue_var_store(&ra_kb, page, count);91 if (ret < 0)92 return ret;93 disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);94 return ret;95}96 97#define QUEUE_SYSFS_LIMIT_SHOW(_field) \98static ssize_t queue_##_field##_show(struct gendisk *disk, char *page) \99{ \100 return queue_var_show(disk->queue->limits._field, page); \101}102 103QUEUE_SYSFS_LIMIT_SHOW(max_segments)104QUEUE_SYSFS_LIMIT_SHOW(max_discard_segments)105QUEUE_SYSFS_LIMIT_SHOW(max_integrity_segments)106QUEUE_SYSFS_LIMIT_SHOW(max_segment_size)107QUEUE_SYSFS_LIMIT_SHOW(logical_block_size)108QUEUE_SYSFS_LIMIT_SHOW(physical_block_size)109QUEUE_SYSFS_LIMIT_SHOW(chunk_sectors)110QUEUE_SYSFS_LIMIT_SHOW(io_min)111QUEUE_SYSFS_LIMIT_SHOW(io_opt)112QUEUE_SYSFS_LIMIT_SHOW(discard_granularity)113QUEUE_SYSFS_LIMIT_SHOW(zone_write_granularity)114QUEUE_SYSFS_LIMIT_SHOW(virt_boundary_mask)115QUEUE_SYSFS_LIMIT_SHOW(dma_alignment)116QUEUE_SYSFS_LIMIT_SHOW(max_open_zones)117QUEUE_SYSFS_LIMIT_SHOW(max_active_zones)118QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_min)119QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_max)120 121#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(_field) \122static ssize_t queue_##_field##_show(struct gendisk *disk, char *page) \123{ \124 return sprintf(page, "%llu\n", \125 (unsigned long long)disk->queue->limits._field << \126 SECTOR_SHIFT); \127}128 129QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_discard_sectors)130QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_discard_sectors)131QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_write_zeroes_sectors)132QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_max_sectors)133QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_boundary_sectors)134 135#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(_field) \136static ssize_t queue_##_field##_show(struct gendisk *disk, char *page) \137{ \138 return queue_var_show(disk->queue->limits._field >> 1, page); \139}140 141QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_sectors)142QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_hw_sectors)143 144#define QUEUE_SYSFS_SHOW_CONST(_name, _val) \145static ssize_t queue_##_name##_show(struct gendisk *disk, char *page) \146{ \147 return sprintf(page, "%d\n", _val); \148}149 150/* deprecated fields */151QUEUE_SYSFS_SHOW_CONST(discard_zeroes_data, 0)152QUEUE_SYSFS_SHOW_CONST(write_same_max, 0)153QUEUE_SYSFS_SHOW_CONST(poll_delay, -1)154 155static ssize_t queue_max_discard_sectors_store(struct gendisk *disk,156 const char *page, size_t count)157{158 unsigned long max_discard_bytes;159 struct queue_limits lim;160 ssize_t ret;161 int err;162 163 ret = queue_var_store(&max_discard_bytes, page, count);164 if (ret < 0)165 return ret;166 167 if (max_discard_bytes & (disk->queue->limits.discard_granularity - 1))168 return -EINVAL;169 170 if ((max_discard_bytes >> SECTOR_SHIFT) > UINT_MAX)171 return -EINVAL;172 173 lim = queue_limits_start_update(disk->queue);174 lim.max_user_discard_sectors = max_discard_bytes >> SECTOR_SHIFT;175 err = queue_limits_commit_update(disk->queue, &lim);176 if (err)177 return err;178 return ret;179}180 181/*182 * For zone append queue_max_zone_append_sectors does not just return the183 * underlying queue limits, but actually contains a calculation. Because of184 * that we can't simply use QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES here.185 */186static ssize_t queue_zone_append_max_show(struct gendisk *disk, char *page)187{188 return sprintf(page, "%llu\n",189 (u64)queue_max_zone_append_sectors(disk->queue) <<190 SECTOR_SHIFT);191}192 193static ssize_t194queue_max_sectors_store(struct gendisk *disk, const char *page, size_t count)195{196 unsigned long max_sectors_kb;197 struct queue_limits lim;198 ssize_t ret;199 int err;200 201 ret = queue_var_store(&max_sectors_kb, page, count);202 if (ret < 0)203 return ret;204 205 lim = queue_limits_start_update(disk->queue);206 lim.max_user_sectors = max_sectors_kb << 1;207 err = queue_limits_commit_update(disk->queue, &lim);208 if (err)209 return err;210 return ret;211}212 213static ssize_t queue_feature_store(struct gendisk *disk, const char *page,214 size_t count, blk_features_t feature)215{216 struct queue_limits lim;217 unsigned long val;218 ssize_t ret;219 220 ret = queue_var_store(&val, page, count);221 if (ret < 0)222 return ret;223 224 lim = queue_limits_start_update(disk->queue);225 if (val)226 lim.features |= feature;227 else228 lim.features &= ~feature;229 ret = queue_limits_commit_update(disk->queue, &lim);230 if (ret)231 return ret;232 return count;233}234 235#define QUEUE_SYSFS_FEATURE(_name, _feature) \236static ssize_t queue_##_name##_show(struct gendisk *disk, char *page) \237{ \238 return sprintf(page, "%u\n", \239 !!(disk->queue->limits.features & _feature)); \240} \241static ssize_t queue_##_name##_store(struct gendisk *disk, \242 const char *page, size_t count) \243{ \244 return queue_feature_store(disk, page, count, _feature); \245}246 247QUEUE_SYSFS_FEATURE(rotational, BLK_FEAT_ROTATIONAL)248QUEUE_SYSFS_FEATURE(add_random, BLK_FEAT_ADD_RANDOM)249QUEUE_SYSFS_FEATURE(iostats, BLK_FEAT_IO_STAT)250QUEUE_SYSFS_FEATURE(stable_writes, BLK_FEAT_STABLE_WRITES);251 252#define QUEUE_SYSFS_FEATURE_SHOW(_name, _feature) \253static ssize_t queue_##_name##_show(struct gendisk *disk, char *page) \254{ \255 return sprintf(page, "%u\n", \256 !!(disk->queue->limits.features & _feature)); \257}258 259QUEUE_SYSFS_FEATURE_SHOW(poll, BLK_FEAT_POLL);260QUEUE_SYSFS_FEATURE_SHOW(fua, BLK_FEAT_FUA);261QUEUE_SYSFS_FEATURE_SHOW(dax, BLK_FEAT_DAX);262 263static ssize_t queue_zoned_show(struct gendisk *disk, char *page)264{265 if (blk_queue_is_zoned(disk->queue))266 return sprintf(page, "host-managed\n");267 return sprintf(page, "none\n");268}269 270static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page)271{272 return queue_var_show(disk_nr_zones(disk), page);273}274 275static ssize_t queue_nomerges_show(struct gendisk *disk, char *page)276{277 return queue_var_show((blk_queue_nomerges(disk->queue) << 1) |278 blk_queue_noxmerges(disk->queue), page);279}280 281static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page,282 size_t count)283{284 unsigned long nm;285 ssize_t ret = queue_var_store(&nm, page, count);286 287 if (ret < 0)288 return ret;289 290 blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, disk->queue);291 blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, disk->queue);292 if (nm == 2)293 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, disk->queue);294 else if (nm)295 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, disk->queue);296 297 return ret;298}299 300static ssize_t queue_rq_affinity_show(struct gendisk *disk, char *page)301{302 bool set = test_bit(QUEUE_FLAG_SAME_COMP, &disk->queue->queue_flags);303 bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &disk->queue->queue_flags);304 305 return queue_var_show(set << force, page);306}307 308static ssize_t309queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count)310{311 ssize_t ret = -EINVAL;312#ifdef CONFIG_SMP313 struct request_queue *q = disk->queue;314 unsigned long val;315 316 ret = queue_var_store(&val, page, count);317 if (ret < 0)318 return ret;319 320 if (val == 2) {321 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);322 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);323 } else if (val == 1) {324 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);325 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);326 } else if (val == 0) {327 blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);328 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);329 }330#endif331 return ret;332}333 334static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page,335 size_t count)336{337 return count;338}339 340static ssize_t queue_poll_store(struct gendisk *disk, const char *page,341 size_t count)342{343 if (!(disk->queue->limits.features & BLK_FEAT_POLL))344 return -EINVAL;345 pr_info_ratelimited("writes to the poll attribute are ignored.\n");346 pr_info_ratelimited("please use driver specific parameters instead.\n");347 return count;348}349 350static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page)351{352 return sprintf(page, "%u\n", jiffies_to_msecs(disk->queue->rq_timeout));353}354 355static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page,356 size_t count)357{358 unsigned int val;359 int err;360 361 err = kstrtou32(page, 10, &val);362 if (err || val == 0)363 return -EINVAL;364 365 blk_queue_rq_timeout(disk->queue, msecs_to_jiffies(val));366 367 return count;368}369 370static ssize_t queue_wc_show(struct gendisk *disk, char *page)371{372 if (blk_queue_write_cache(disk->queue))373 return sprintf(page, "write back\n");374 return sprintf(page, "write through\n");375}376 377static ssize_t queue_wc_store(struct gendisk *disk, const char *page,378 size_t count)379{380 struct queue_limits lim;381 bool disable;382 int err;383 384 if (!strncmp(page, "write back", 10)) {385 disable = false;386 } else if (!strncmp(page, "write through", 13) ||387 !strncmp(page, "none", 4)) {388 disable = true;389 } else {390 return -EINVAL;391 }392 393 lim = queue_limits_start_update(disk->queue);394 if (disable)395 lim.flags |= BLK_FLAG_WRITE_CACHE_DISABLED;396 else397 lim.flags &= ~BLK_FLAG_WRITE_CACHE_DISABLED;398 err = queue_limits_commit_update(disk->queue, &lim);399 if (err)400 return err;401 return count;402}403 404#define QUEUE_RO_ENTRY(_prefix, _name) \405static struct queue_sysfs_entry _prefix##_entry = { \406 .attr = { .name = _name, .mode = 0444 }, \407 .show = _prefix##_show, \408};409 410#define QUEUE_RW_ENTRY(_prefix, _name) \411static struct queue_sysfs_entry _prefix##_entry = { \412 .attr = { .name = _name, .mode = 0644 }, \413 .show = _prefix##_show, \414 .store = _prefix##_store, \415};416 417#define QUEUE_RW_LOAD_MODULE_ENTRY(_prefix, _name) \418static struct queue_sysfs_entry _prefix##_entry = { \419 .attr = { .name = _name, .mode = 0644 }, \420 .show = _prefix##_show, \421 .load_module = _prefix##_load_module, \422 .store = _prefix##_store, \423}424 425QUEUE_RW_ENTRY(queue_requests, "nr_requests");426QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");427QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");428QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");429QUEUE_RO_ENTRY(queue_max_segments, "max_segments");430QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");431QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");432QUEUE_RW_LOAD_MODULE_ENTRY(elv_iosched, "scheduler");433 434QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");435QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");436QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");437QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");438QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");439 440QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");441QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");442QUEUE_RO_ENTRY(queue_max_hw_discard_sectors, "discard_max_hw_bytes");443QUEUE_RW_ENTRY(queue_max_discard_sectors, "discard_max_bytes");444QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");445 446QUEUE_RO_ENTRY(queue_atomic_write_max_sectors, "atomic_write_max_bytes");447QUEUE_RO_ENTRY(queue_atomic_write_boundary_sectors,448 "atomic_write_boundary_bytes");449QUEUE_RO_ENTRY(queue_atomic_write_unit_max, "atomic_write_unit_max_bytes");450QUEUE_RO_ENTRY(queue_atomic_write_unit_min, "atomic_write_unit_min_bytes");451 452QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");453QUEUE_RO_ENTRY(queue_max_write_zeroes_sectors, "write_zeroes_max_bytes");454QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");455QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");456 457QUEUE_RO_ENTRY(queue_zoned, "zoned");458QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");459QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");460QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");461 462QUEUE_RW_ENTRY(queue_nomerges, "nomerges");463QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");464QUEUE_RW_ENTRY(queue_poll, "io_poll");465QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");466QUEUE_RW_ENTRY(queue_wc, "write_cache");467QUEUE_RO_ENTRY(queue_fua, "fua");468QUEUE_RO_ENTRY(queue_dax, "dax");469QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");470QUEUE_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");471QUEUE_RO_ENTRY(queue_dma_alignment, "dma_alignment");472 473/* legacy alias for logical_block_size: */474static struct queue_sysfs_entry queue_hw_sector_size_entry = {475 .attr = {.name = "hw_sector_size", .mode = 0444 },476 .show = queue_logical_block_size_show,477};478 479QUEUE_RW_ENTRY(queue_rotational, "rotational");480QUEUE_RW_ENTRY(queue_iostats, "iostats");481QUEUE_RW_ENTRY(queue_add_random, "add_random");482QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");483 484#ifdef CONFIG_BLK_WBT485static ssize_t queue_var_store64(s64 *var, const char *page)486{487 int err;488 s64 v;489 490 err = kstrtos64(page, 10, &v);491 if (err < 0)492 return err;493 494 *var = v;495 return 0;496}497 498static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)499{500 if (!wbt_rq_qos(disk->queue))501 return -EINVAL;502 503 if (wbt_disabled(disk->queue))504 return sprintf(page, "0\n");505 506 return sprintf(page, "%llu\n",507 div_u64(wbt_get_min_lat(disk->queue), 1000));508}509 510static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,511 size_t count)512{513 struct request_queue *q = disk->queue;514 struct rq_qos *rqos;515 ssize_t ret;516 s64 val;517 518 ret = queue_var_store64(&val, page);519 if (ret < 0)520 return ret;521 if (val < -1)522 return -EINVAL;523 524 rqos = wbt_rq_qos(q);525 if (!rqos) {526 ret = wbt_init(disk);527 if (ret)528 return ret;529 }530 531 if (val == -1)532 val = wbt_default_latency_nsec(q);533 else if (val >= 0)534 val *= 1000ULL;535 536 if (wbt_get_min_lat(q) == val)537 return count;538 539 /*540 * Ensure that the queue is idled, in case the latency update541 * ends up either enabling or disabling wbt completely. We can't542 * have IO inflight if that happens.543 */544 blk_mq_quiesce_queue(q);545 546 wbt_set_min_lat(q, val);547 548 blk_mq_unquiesce_queue(q);549 550 return count;551}552 553QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");554#endif555 556/* Common attributes for bio-based and request-based queues. */557static struct attribute *queue_attrs[] = {558 &queue_ra_entry.attr,559 &queue_max_hw_sectors_entry.attr,560 &queue_max_sectors_entry.attr,561 &queue_max_segments_entry.attr,562 &queue_max_discard_segments_entry.attr,563 &queue_max_integrity_segments_entry.attr,564 &queue_max_segment_size_entry.attr,565 &queue_hw_sector_size_entry.attr,566 &queue_logical_block_size_entry.attr,567 &queue_physical_block_size_entry.attr,568 &queue_chunk_sectors_entry.attr,569 &queue_io_min_entry.attr,570 &queue_io_opt_entry.attr,571 &queue_discard_granularity_entry.attr,572 &queue_max_discard_sectors_entry.attr,573 &queue_max_hw_discard_sectors_entry.attr,574 &queue_discard_zeroes_data_entry.attr,575 &queue_atomic_write_max_sectors_entry.attr,576 &queue_atomic_write_boundary_sectors_entry.attr,577 &queue_atomic_write_unit_min_entry.attr,578 &queue_atomic_write_unit_max_entry.attr,579 &queue_write_same_max_entry.attr,580 &queue_max_write_zeroes_sectors_entry.attr,581 &queue_zone_append_max_entry.attr,582 &queue_zone_write_granularity_entry.attr,583 &queue_rotational_entry.attr,584 &queue_zoned_entry.attr,585 &queue_nr_zones_entry.attr,586 &queue_max_open_zones_entry.attr,587 &queue_max_active_zones_entry.attr,588 &queue_nomerges_entry.attr,589 &queue_iostats_entry.attr,590 &queue_stable_writes_entry.attr,591 &queue_add_random_entry.attr,592 &queue_poll_entry.attr,593 &queue_wc_entry.attr,594 &queue_fua_entry.attr,595 &queue_dax_entry.attr,596 &queue_poll_delay_entry.attr,597 &queue_virt_boundary_mask_entry.attr,598 &queue_dma_alignment_entry.attr,599 NULL,600};601 602/* Request-based queue attributes that are not relevant for bio-based queues. */603static struct attribute *blk_mq_queue_attrs[] = {604 &queue_requests_entry.attr,605 &elv_iosched_entry.attr,606 &queue_rq_affinity_entry.attr,607 &queue_io_timeout_entry.attr,608#ifdef CONFIG_BLK_WBT609 &queue_wb_lat_entry.attr,610#endif611 NULL,612};613 614static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,615 int n)616{617 struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);618 struct request_queue *q = disk->queue;619 620 if ((attr == &queue_max_open_zones_entry.attr ||621 attr == &queue_max_active_zones_entry.attr) &&622 !blk_queue_is_zoned(q))623 return 0;624 625 return attr->mode;626}627 628static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,629 struct attribute *attr, int n)630{631 struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);632 struct request_queue *q = disk->queue;633 634 if (!queue_is_mq(q))635 return 0;636 637 if (attr == &queue_io_timeout_entry.attr && !q->mq_ops->timeout)638 return 0;639 640 return attr->mode;641}642 643static struct attribute_group queue_attr_group = {644 .attrs = queue_attrs,645 .is_visible = queue_attr_visible,646};647 648static struct attribute_group blk_mq_queue_attr_group = {649 .attrs = blk_mq_queue_attrs,650 .is_visible = blk_mq_queue_attr_visible,651};652 653#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)654 655static ssize_t656queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)657{658 struct queue_sysfs_entry *entry = to_queue(attr);659 struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);660 ssize_t res;661 662 if (!entry->show)663 return -EIO;664 mutex_lock(&disk->queue->sysfs_lock);665 res = entry->show(disk, page);666 mutex_unlock(&disk->queue->sysfs_lock);667 return res;668}669 670static ssize_t671queue_attr_store(struct kobject *kobj, struct attribute *attr,672 const char *page, size_t length)673{674 struct queue_sysfs_entry *entry = to_queue(attr);675 struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);676 struct request_queue *q = disk->queue;677 ssize_t res;678 679 if (!entry->store)680 return -EIO;681 682 /*683 * If the attribute needs to load a module, do it before freezing the684 * queue to ensure that the module file can be read when the request685 * queue is the one for the device storing the module file.686 */687 if (entry->load_module) {688 res = entry->load_module(disk, page, length);689 if (res)690 return res;691 }692 693 blk_mq_freeze_queue(q);694 mutex_lock(&q->sysfs_lock);695 res = entry->store(disk, page, length);696 mutex_unlock(&q->sysfs_lock);697 blk_mq_unfreeze_queue(q);698 return res;699}700 701static const struct sysfs_ops queue_sysfs_ops = {702 .show = queue_attr_show,703 .store = queue_attr_store,704};705 706static const struct attribute_group *blk_queue_attr_groups[] = {707 &queue_attr_group,708 &blk_mq_queue_attr_group,709 NULL710};711 712static void blk_queue_release(struct kobject *kobj)713{714 /* nothing to do here, all data is associated with the parent gendisk */715}716 717static const struct kobj_type blk_queue_ktype = {718 .default_groups = blk_queue_attr_groups,719 .sysfs_ops = &queue_sysfs_ops,720 .release = blk_queue_release,721};722 723static void blk_debugfs_remove(struct gendisk *disk)724{725 struct request_queue *q = disk->queue;726 727 mutex_lock(&q->debugfs_mutex);728 blk_trace_shutdown(q);729 debugfs_remove_recursive(q->debugfs_dir);730 q->debugfs_dir = NULL;731 q->sched_debugfs_dir = NULL;732 q->rqos_debugfs_dir = NULL;733 mutex_unlock(&q->debugfs_mutex);734}735 736/**737 * blk_register_queue - register a block layer queue with sysfs738 * @disk: Disk of which the request queue should be registered with sysfs.739 */740int blk_register_queue(struct gendisk *disk)741{742 struct request_queue *q = disk->queue;743 int ret;744 745 mutex_lock(&q->sysfs_dir_lock);746 kobject_init(&disk->queue_kobj, &blk_queue_ktype);747 ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue");748 if (ret < 0)749 goto out_put_queue_kobj;750 751 if (queue_is_mq(q)) {752 ret = blk_mq_sysfs_register(disk);753 if (ret)754 goto out_put_queue_kobj;755 }756 mutex_lock(&q->sysfs_lock);757 758 mutex_lock(&q->debugfs_mutex);759 q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root);760 if (queue_is_mq(q))761 blk_mq_debugfs_register(q);762 mutex_unlock(&q->debugfs_mutex);763 764 ret = disk_register_independent_access_ranges(disk);765 if (ret)766 goto out_debugfs_remove;767 768 if (q->elevator) {769 ret = elv_register_queue(q, false);770 if (ret)771 goto out_unregister_ia_ranges;772 }773 774 ret = blk_crypto_sysfs_register(disk);775 if (ret)776 goto out_elv_unregister;777 778 blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);779 wbt_enable_default(disk);780 781 /* Now everything is ready and send out KOBJ_ADD uevent */782 kobject_uevent(&disk->queue_kobj, KOBJ_ADD);783 if (q->elevator)784 kobject_uevent(&q->elevator->kobj, KOBJ_ADD);785 mutex_unlock(&q->sysfs_lock);786 mutex_unlock(&q->sysfs_dir_lock);787 788 /*789 * SCSI probing may synchronously create and destroy a lot of790 * request_queues for non-existent devices. Shutting down a fully791 * functional queue takes measureable wallclock time as RCU grace792 * periods are involved. To avoid excessive latency in these793 * cases, a request_queue starts out in a degraded mode which is794 * faster to shut down and is made fully functional here as795 * request_queues for non-existent devices never get registered.796 */797 if (!blk_queue_init_done(q)) {798 blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);799 percpu_ref_switch_to_percpu(&q->q_usage_counter);800 }801 802 return ret;803 804out_elv_unregister:805 elv_unregister_queue(q);806out_unregister_ia_ranges:807 disk_unregister_independent_access_ranges(disk);808out_debugfs_remove:809 blk_debugfs_remove(disk);810 mutex_unlock(&q->sysfs_lock);811out_put_queue_kobj:812 kobject_put(&disk->queue_kobj);813 mutex_unlock(&q->sysfs_dir_lock);814 return ret;815}816 817/**818 * blk_unregister_queue - counterpart of blk_register_queue()819 * @disk: Disk of which the request queue should be unregistered from sysfs.820 *821 * Note: the caller is responsible for guaranteeing that this function is called822 * after blk_register_queue() has finished.823 */824void blk_unregister_queue(struct gendisk *disk)825{826 struct request_queue *q = disk->queue;827 828 if (WARN_ON(!q))829 return;830 831 /* Return early if disk->queue was never registered. */832 if (!blk_queue_registered(q))833 return;834 835 /*836 * Since sysfs_remove_dir() prevents adding new directory entries837 * before removal of existing entries starts, protect against838 * concurrent elv_iosched_store() calls.839 */840 mutex_lock(&q->sysfs_lock);841 blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);842 mutex_unlock(&q->sysfs_lock);843 844 mutex_lock(&q->sysfs_dir_lock);845 /*846 * Remove the sysfs attributes before unregistering the queue data847 * structures that can be modified through sysfs.848 */849 if (queue_is_mq(q))850 blk_mq_sysfs_unregister(disk);851 blk_crypto_sysfs_unregister(disk);852 853 mutex_lock(&q->sysfs_lock);854 elv_unregister_queue(q);855 disk_unregister_independent_access_ranges(disk);856 mutex_unlock(&q->sysfs_lock);857 858 /* Now that we've deleted all child objects, we can delete the queue. */859 kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE);860 kobject_del(&disk->queue_kobj);861 mutex_unlock(&q->sysfs_dir_lock);862 863 blk_debugfs_remove(disk);864}865