2205 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Common Block IO controller cgroup interface4 *5 * Based on ideas and code from CFQ, CFS and BFQ:6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>7 *8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>9 * Paolo Valente <paolo.valente@unimore.it>10 *11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>12 * Nauman Rafique <nauman@google.com>13 *14 * For policy-specific per-blkcg data:15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>16 * Arianna Avanzini <avanzini.arianna@gmail.com>17 */18#include <linux/ioprio.h>19#include <linux/kdev_t.h>20#include <linux/module.h>21#include <linux/sched/signal.h>22#include <linux/err.h>23#include <linux/blkdev.h>24#include <linux/backing-dev.h>25#include <linux/slab.h>26#include <linux/delay.h>27#include <linux/atomic.h>28#include <linux/ctype.h>29#include <linux/resume_user_mode.h>30#include <linux/psi.h>31#include <linux/part_stat.h>32#include "blk.h"33#include "blk-cgroup.h"34#include "blk-ioprio.h"35#include "blk-throttle.h"36 37static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);38 39/*40 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.41 * blkcg_pol_register_mutex nests outside of it and synchronizes entire42 * policy [un]register operations including cgroup file additions /43 * removals. Putting cgroup file registration outside blkcg_pol_mutex44 * allows grabbing it from cgroup callbacks.45 */46static DEFINE_MUTEX(blkcg_pol_register_mutex);47static DEFINE_MUTEX(blkcg_pol_mutex);48 49struct blkcg blkcg_root;50EXPORT_SYMBOL_GPL(blkcg_root);51 52struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;53EXPORT_SYMBOL_GPL(blkcg_root_css);54 55static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];56 57static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */58 59bool blkcg_debug_stats = false;60 61static DEFINE_RAW_SPINLOCK(blkg_stat_lock);62 63#define BLKG_DESTROY_BATCH_SIZE 6464 65/*66 * Lockless lists for tracking IO stats update67 *68 * New IO stats are stored in the percpu iostat_cpu within blkcg_gq (blkg).69 * There are multiple blkg's (one for each block device) attached to each70 * blkcg. The rstat code keeps track of which cpu has IO stats updated,71 * but it doesn't know which blkg has the updated stats. If there are many72 * block devices in a system, the cost of iterating all the blkg's to flush73 * out the IO stats can be high. To reduce such overhead, a set of percpu74 * lockless lists (lhead) per blkcg are used to track the set of recently75 * updated iostat_cpu's since the last flush. An iostat_cpu will be put76 * onto the lockless list on the update side [blk_cgroup_bio_start()] if77 * not there yet and then removed when being flushed [blkcg_rstat_flush()].78 * References to blkg are gotten and then put back in the process to79 * protect against blkg removal.80 *81 * Return: 0 if successful or -ENOMEM if allocation fails.82 */83static int init_blkcg_llists(struct blkcg *blkcg)84{85 int cpu;86 87 blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);88 if (!blkcg->lhead)89 return -ENOMEM;90 91 for_each_possible_cpu(cpu)92 init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));93 return 0;94}95 96/**97 * blkcg_css - find the current css98 *99 * Find the css associated with either the kthread or the current task.100 * This may return a dying css, so it is up to the caller to use tryget logic101 * to confirm it is alive and well.102 */103static struct cgroup_subsys_state *blkcg_css(void)104{105 struct cgroup_subsys_state *css;106 107 css = kthread_blkcg();108 if (css)109 return css;110 return task_css(current, io_cgrp_id);111}112 113static bool blkcg_policy_enabled(struct request_queue *q,114 const struct blkcg_policy *pol)115{116 return pol && test_bit(pol->plid, q->blkcg_pols);117}118 119static void blkg_free_workfn(struct work_struct *work)120{121 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,122 free_work);123 struct request_queue *q = blkg->q;124 int i;125 126 /*127 * pd_free_fn() can also be called from blkcg_deactivate_policy(),128 * in order to make sure pd_free_fn() is called in order, the deletion129 * of the list blkg->q_node is delayed to here from blkg_destroy(), and130 * blkcg_mutex is used to synchronize blkg_free_workfn() and131 * blkcg_deactivate_policy().132 */133 mutex_lock(&q->blkcg_mutex);134 for (i = 0; i < BLKCG_MAX_POLS; i++)135 if (blkg->pd[i])136 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);137 if (blkg->parent)138 blkg_put(blkg->parent);139 spin_lock_irq(&q->queue_lock);140 list_del_init(&blkg->q_node);141 spin_unlock_irq(&q->queue_lock);142 mutex_unlock(&q->blkcg_mutex);143 144 blk_put_queue(q);145 free_percpu(blkg->iostat_cpu);146 percpu_ref_exit(&blkg->refcnt);147 kfree(blkg);148}149 150/**151 * blkg_free - free a blkg152 * @blkg: blkg to free153 *154 * Free @blkg which may be partially allocated.155 */156static void blkg_free(struct blkcg_gq *blkg)157{158 if (!blkg)159 return;160 161 /*162 * Both ->pd_free_fn() and request queue's release handler may163 * sleep, so free us by scheduling one work func164 */165 INIT_WORK(&blkg->free_work, blkg_free_workfn);166 schedule_work(&blkg->free_work);167}168 169static void __blkg_release(struct rcu_head *rcu)170{171 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);172 struct blkcg *blkcg = blkg->blkcg;173 int cpu;174 175#ifdef CONFIG_BLK_CGROUP_PUNT_BIO176 WARN_ON(!bio_list_empty(&blkg->async_bios));177#endif178 /*179 * Flush all the non-empty percpu lockless lists before releasing180 * us, given these stat belongs to us.181 *182 * blkg_stat_lock is for serializing blkg stat update183 */184 for_each_possible_cpu(cpu)185 __blkcg_rstat_flush(blkcg, cpu);186 187 /* release the blkcg and parent blkg refs this blkg has been holding */188 css_put(&blkg->blkcg->css);189 blkg_free(blkg);190}191 192/*193 * A group is RCU protected, but having an rcu lock does not mean that one194 * can access all the fields of blkg and assume these are valid. For195 * example, don't try to follow throtl_data and request queue links.196 *197 * Having a reference to blkg under an rcu allows accesses to only values198 * local to groups like group stats and group rate limits.199 */200static void blkg_release(struct percpu_ref *ref)201{202 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);203 204 call_rcu(&blkg->rcu_head, __blkg_release);205}206 207#ifdef CONFIG_BLK_CGROUP_PUNT_BIO208static struct workqueue_struct *blkcg_punt_bio_wq;209 210static void blkg_async_bio_workfn(struct work_struct *work)211{212 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,213 async_bio_work);214 struct bio_list bios = BIO_EMPTY_LIST;215 struct bio *bio;216 struct blk_plug plug;217 bool need_plug = false;218 219 /* as long as there are pending bios, @blkg can't go away */220 spin_lock(&blkg->async_bio_lock);221 bio_list_merge_init(&bios, &blkg->async_bios);222 spin_unlock(&blkg->async_bio_lock);223 224 /* start plug only when bio_list contains at least 2 bios */225 if (bios.head && bios.head->bi_next) {226 need_plug = true;227 blk_start_plug(&plug);228 }229 while ((bio = bio_list_pop(&bios)))230 submit_bio(bio);231 if (need_plug)232 blk_finish_plug(&plug);233}234 235/*236 * When a shared kthread issues a bio for a cgroup, doing so synchronously can237 * lead to priority inversions as the kthread can be trapped waiting for that238 * cgroup. Use this helper instead of submit_bio to punt the actual issuing to239 * a dedicated per-blkcg work item to avoid such priority inversions.240 */241void blkcg_punt_bio_submit(struct bio *bio)242{243 struct blkcg_gq *blkg = bio->bi_blkg;244 245 if (blkg->parent) {246 spin_lock(&blkg->async_bio_lock);247 bio_list_add(&blkg->async_bios, bio);248 spin_unlock(&blkg->async_bio_lock);249 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);250 } else {251 /* never bounce for the root cgroup */252 submit_bio(bio);253 }254}255EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit);256 257static int __init blkcg_punt_bio_init(void)258{259 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",260 WQ_MEM_RECLAIM | WQ_FREEZABLE |261 WQ_UNBOUND | WQ_SYSFS, 0);262 if (!blkcg_punt_bio_wq)263 return -ENOMEM;264 return 0;265}266subsys_initcall(blkcg_punt_bio_init);267#endif /* CONFIG_BLK_CGROUP_PUNT_BIO */268 269/**270 * bio_blkcg_css - return the blkcg CSS associated with a bio271 * @bio: target bio272 *273 * This returns the CSS for the blkcg associated with a bio, or %NULL if not274 * associated. Callers are expected to either handle %NULL or know association275 * has been done prior to calling this.276 */277struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)278{279 if (!bio || !bio->bi_blkg)280 return NULL;281 return &bio->bi_blkg->blkcg->css;282}283EXPORT_SYMBOL_GPL(bio_blkcg_css);284 285/**286 * blkcg_parent - get the parent of a blkcg287 * @blkcg: blkcg of interest288 *289 * Return the parent blkcg of @blkcg. Can be called anytime.290 */291static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)292{293 return css_to_blkcg(blkcg->css.parent);294}295 296/**297 * blkg_alloc - allocate a blkg298 * @blkcg: block cgroup the new blkg is associated with299 * @disk: gendisk the new blkg is associated with300 * @gfp_mask: allocation mask to use301 *302 * Allocate a new blkg associating @blkcg and @disk.303 */304static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,305 gfp_t gfp_mask)306{307 struct blkcg_gq *blkg;308 int i, cpu;309 310 /* alloc and init base part */311 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node);312 if (!blkg)313 return NULL;314 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))315 goto out_free_blkg;316 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);317 if (!blkg->iostat_cpu)318 goto out_exit_refcnt;319 if (!blk_get_queue(disk->queue))320 goto out_free_iostat;321 322 blkg->q = disk->queue;323 INIT_LIST_HEAD(&blkg->q_node);324 blkg->blkcg = blkcg;325 blkg->iostat.blkg = blkg;326#ifdef CONFIG_BLK_CGROUP_PUNT_BIO327 spin_lock_init(&blkg->async_bio_lock);328 bio_list_init(&blkg->async_bios);329 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);330#endif331 332 u64_stats_init(&blkg->iostat.sync);333 for_each_possible_cpu(cpu) {334 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);335 per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;336 }337 338 for (i = 0; i < BLKCG_MAX_POLS; i++) {339 struct blkcg_policy *pol = blkcg_policy[i];340 struct blkg_policy_data *pd;341 342 if (!blkcg_policy_enabled(disk->queue, pol))343 continue;344 345 /* alloc per-policy data and attach it to blkg */346 pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);347 if (!pd)348 goto out_free_pds;349 blkg->pd[i] = pd;350 pd->blkg = blkg;351 pd->plid = i;352 pd->online = false;353 }354 355 return blkg;356 357out_free_pds:358 while (--i >= 0)359 if (blkg->pd[i])360 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);361 blk_put_queue(disk->queue);362out_free_iostat:363 free_percpu(blkg->iostat_cpu);364out_exit_refcnt:365 percpu_ref_exit(&blkg->refcnt);366out_free_blkg:367 kfree(blkg);368 return NULL;369}370 371/*372 * If @new_blkg is %NULL, this function tries to allocate a new one as373 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.374 */375static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,376 struct blkcg_gq *new_blkg)377{378 struct blkcg_gq *blkg;379 int i, ret;380 381 lockdep_assert_held(&disk->queue->queue_lock);382 383 /* request_queue is dying, do not create/recreate a blkg */384 if (blk_queue_dying(disk->queue)) {385 ret = -ENODEV;386 goto err_free_blkg;387 }388 389 /* blkg holds a reference to blkcg */390 if (!css_tryget_online(&blkcg->css)) {391 ret = -ENODEV;392 goto err_free_blkg;393 }394 395 /* allocate */396 if (!new_blkg) {397 new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN);398 if (unlikely(!new_blkg)) {399 ret = -ENOMEM;400 goto err_put_css;401 }402 }403 blkg = new_blkg;404 405 /* link parent */406 if (blkcg_parent(blkcg)) {407 blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);408 if (WARN_ON_ONCE(!blkg->parent)) {409 ret = -ENODEV;410 goto err_put_css;411 }412 blkg_get(blkg->parent);413 }414 415 /* invoke per-policy init */416 for (i = 0; i < BLKCG_MAX_POLS; i++) {417 struct blkcg_policy *pol = blkcg_policy[i];418 419 if (blkg->pd[i] && pol->pd_init_fn)420 pol->pd_init_fn(blkg->pd[i]);421 }422 423 /* insert */424 spin_lock(&blkcg->lock);425 ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);426 if (likely(!ret)) {427 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);428 list_add(&blkg->q_node, &disk->queue->blkg_list);429 430 for (i = 0; i < BLKCG_MAX_POLS; i++) {431 struct blkcg_policy *pol = blkcg_policy[i];432 433 if (blkg->pd[i]) {434 if (pol->pd_online_fn)435 pol->pd_online_fn(blkg->pd[i]);436 blkg->pd[i]->online = true;437 }438 }439 }440 blkg->online = true;441 spin_unlock(&blkcg->lock);442 443 if (!ret)444 return blkg;445 446 /* @blkg failed fully initialized, use the usual release path */447 blkg_put(blkg);448 return ERR_PTR(ret);449 450err_put_css:451 css_put(&blkcg->css);452err_free_blkg:453 if (new_blkg)454 blkg_free(new_blkg);455 return ERR_PTR(ret);456}457 458/**459 * blkg_lookup_create - lookup blkg, try to create one if not there460 * @blkcg: blkcg of interest461 * @disk: gendisk of interest462 *463 * Lookup blkg for the @blkcg - @disk pair. If it doesn't exist, try to464 * create one. blkg creation is performed recursively from blkcg_root such465 * that all non-root blkg's have access to the parent blkg. This function466 * should be called under RCU read lock and takes @disk->queue->queue_lock.467 *468 * Returns the blkg or the closest blkg if blkg_create() fails as it walks469 * down from root.470 */471static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,472 struct gendisk *disk)473{474 struct request_queue *q = disk->queue;475 struct blkcg_gq *blkg;476 unsigned long flags;477 478 WARN_ON_ONCE(!rcu_read_lock_held());479 480 blkg = blkg_lookup(blkcg, q);481 if (blkg)482 return blkg;483 484 spin_lock_irqsave(&q->queue_lock, flags);485 blkg = blkg_lookup(blkcg, q);486 if (blkg) {487 if (blkcg != &blkcg_root &&488 blkg != rcu_dereference(blkcg->blkg_hint))489 rcu_assign_pointer(blkcg->blkg_hint, blkg);490 goto found;491 }492 493 /*494 * Create blkgs walking down from blkcg_root to @blkcg, so that all495 * non-root blkgs have access to their parents. Returns the closest496 * blkg to the intended blkg should blkg_create() fail.497 */498 while (true) {499 struct blkcg *pos = blkcg;500 struct blkcg *parent = blkcg_parent(blkcg);501 struct blkcg_gq *ret_blkg = q->root_blkg;502 503 while (parent) {504 blkg = blkg_lookup(parent, q);505 if (blkg) {506 /* remember closest blkg */507 ret_blkg = blkg;508 break;509 }510 pos = parent;511 parent = blkcg_parent(parent);512 }513 514 blkg = blkg_create(pos, disk, NULL);515 if (IS_ERR(blkg)) {516 blkg = ret_blkg;517 break;518 }519 if (pos == blkcg)520 break;521 }522 523found:524 spin_unlock_irqrestore(&q->queue_lock, flags);525 return blkg;526}527 528static void blkg_destroy(struct blkcg_gq *blkg)529{530 struct blkcg *blkcg = blkg->blkcg;531 int i;532 533 lockdep_assert_held(&blkg->q->queue_lock);534 lockdep_assert_held(&blkcg->lock);535 536 /*537 * blkg stays on the queue list until blkg_free_workfn(), see details in538 * blkg_free_workfn(), hence this function can be called from539 * blkcg_destroy_blkgs() first and again from blkg_destroy_all() before540 * blkg_free_workfn().541 */542 if (hlist_unhashed(&blkg->blkcg_node))543 return;544 545 for (i = 0; i < BLKCG_MAX_POLS; i++) {546 struct blkcg_policy *pol = blkcg_policy[i];547 548 if (blkg->pd[i] && blkg->pd[i]->online) {549 blkg->pd[i]->online = false;550 if (pol->pd_offline_fn)551 pol->pd_offline_fn(blkg->pd[i]);552 }553 }554 555 blkg->online = false;556 557 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);558 hlist_del_init_rcu(&blkg->blkcg_node);559 560 /*561 * Both setting lookup hint to and clearing it from @blkg are done562 * under queue_lock. If it's not pointing to @blkg now, it never563 * will. Hint assignment itself can race safely.564 */565 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)566 rcu_assign_pointer(blkcg->blkg_hint, NULL);567 568 /*569 * Put the reference taken at the time of creation so that when all570 * queues are gone, group can be destroyed.571 */572 percpu_ref_kill(&blkg->refcnt);573}574 575static void blkg_destroy_all(struct gendisk *disk)576{577 struct request_queue *q = disk->queue;578 struct blkcg_gq *blkg;579 int count = BLKG_DESTROY_BATCH_SIZE;580 int i;581 582restart:583 spin_lock_irq(&q->queue_lock);584 list_for_each_entry(blkg, &q->blkg_list, q_node) {585 struct blkcg *blkcg = blkg->blkcg;586 587 if (hlist_unhashed(&blkg->blkcg_node))588 continue;589 590 spin_lock(&blkcg->lock);591 blkg_destroy(blkg);592 spin_unlock(&blkcg->lock);593 594 /*595 * in order to avoid holding the spin lock for too long, release596 * it when a batch of blkgs are destroyed.597 */598 if (!(--count)) {599 count = BLKG_DESTROY_BATCH_SIZE;600 spin_unlock_irq(&q->queue_lock);601 cond_resched();602 goto restart;603 }604 }605 606 /*607 * Mark policy deactivated since policy offline has been done, and608 * the free is scheduled, so future blkcg_deactivate_policy() can609 * be bypassed610 */611 for (i = 0; i < BLKCG_MAX_POLS; i++) {612 struct blkcg_policy *pol = blkcg_policy[i];613 614 if (pol)615 __clear_bit(pol->plid, q->blkcg_pols);616 }617 618 q->root_blkg = NULL;619 spin_unlock_irq(&q->queue_lock);620}621 622static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)623{624 int i;625 626 for (i = 0; i < BLKG_IOSTAT_NR; i++) {627 dst->bytes[i] = src->bytes[i];628 dst->ios[i] = src->ios[i];629 }630}631 632static void __blkg_clear_stat(struct blkg_iostat_set *bis)633{634 struct blkg_iostat cur = {0};635 unsigned long flags;636 637 flags = u64_stats_update_begin_irqsave(&bis->sync);638 blkg_iostat_set(&bis->cur, &cur);639 blkg_iostat_set(&bis->last, &cur);640 u64_stats_update_end_irqrestore(&bis->sync, flags);641}642 643static void blkg_clear_stat(struct blkcg_gq *blkg)644{645 int cpu;646 647 for_each_possible_cpu(cpu) {648 struct blkg_iostat_set *s = per_cpu_ptr(blkg->iostat_cpu, cpu);649 650 __blkg_clear_stat(s);651 }652 __blkg_clear_stat(&blkg->iostat);653}654 655static int blkcg_reset_stats(struct cgroup_subsys_state *css,656 struct cftype *cftype, u64 val)657{658 struct blkcg *blkcg = css_to_blkcg(css);659 struct blkcg_gq *blkg;660 int i;661 662 mutex_lock(&blkcg_pol_mutex);663 spin_lock_irq(&blkcg->lock);664 665 /*666 * Note that stat reset is racy - it doesn't synchronize against667 * stat updates. This is a debug feature which shouldn't exist668 * anyway. If you get hit by a race, retry.669 */670 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {671 blkg_clear_stat(blkg);672 for (i = 0; i < BLKCG_MAX_POLS; i++) {673 struct blkcg_policy *pol = blkcg_policy[i];674 675 if (blkg->pd[i] && pol->pd_reset_stats_fn)676 pol->pd_reset_stats_fn(blkg->pd[i]);677 }678 }679 680 spin_unlock_irq(&blkcg->lock);681 mutex_unlock(&blkcg_pol_mutex);682 return 0;683}684 685const char *blkg_dev_name(struct blkcg_gq *blkg)686{687 if (!blkg->q->disk)688 return NULL;689 return bdi_dev_name(blkg->q->disk->bdi);690}691 692/**693 * blkcg_print_blkgs - helper for printing per-blkg data694 * @sf: seq_file to print to695 * @blkcg: blkcg of interest696 * @prfill: fill function to print out a blkg697 * @pol: policy in question698 * @data: data to be passed to @prfill699 * @show_total: to print out sum of prfill return values or not700 *701 * This function invokes @prfill on each blkg of @blkcg if pd for the702 * policy specified by @pol exists. @prfill is invoked with @sf, the703 * policy data and @data and the matching queue lock held. If @show_total704 * is %true, the sum of the return values from @prfill is printed with705 * "Total" label at the end.706 *707 * This is to be used to construct print functions for708 * cftype->read_seq_string method.709 */710void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,711 u64 (*prfill)(struct seq_file *,712 struct blkg_policy_data *, int),713 const struct blkcg_policy *pol, int data,714 bool show_total)715{716 struct blkcg_gq *blkg;717 u64 total = 0;718 719 rcu_read_lock();720 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {721 spin_lock_irq(&blkg->q->queue_lock);722 if (blkcg_policy_enabled(blkg->q, pol))723 total += prfill(sf, blkg->pd[pol->plid], data);724 spin_unlock_irq(&blkg->q->queue_lock);725 }726 rcu_read_unlock();727 728 if (show_total)729 seq_printf(sf, "Total %llu\n", (unsigned long long)total);730}731EXPORT_SYMBOL_GPL(blkcg_print_blkgs);732 733/**734 * __blkg_prfill_u64 - prfill helper for a single u64 value735 * @sf: seq_file to print to736 * @pd: policy private data of interest737 * @v: value to print738 *739 * Print @v to @sf for the device associated with @pd.740 */741u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)742{743 const char *dname = blkg_dev_name(pd->blkg);744 745 if (!dname)746 return 0;747 748 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);749 return v;750}751EXPORT_SYMBOL_GPL(__blkg_prfill_u64);752 753/**754 * blkg_conf_init - initialize a blkg_conf_ctx755 * @ctx: blkg_conf_ctx to initialize756 * @input: input string757 *758 * Initialize @ctx which can be used to parse blkg config input string @input.759 * Once initialized, @ctx can be used with blkg_conf_open_bdev() and760 * blkg_conf_prep(), and must be cleaned up with blkg_conf_exit().761 */762void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input)763{764 *ctx = (struct blkg_conf_ctx){ .input = input };765}766EXPORT_SYMBOL_GPL(blkg_conf_init);767 768/**769 * blkg_conf_open_bdev - parse and open bdev for per-blkg config update770 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()771 *772 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update from773 * @ctx->input and get and store the matching bdev in @ctx->bdev. @ctx->body is774 * set to point past the device node prefix.775 *776 * This function may be called multiple times on @ctx and the extra calls become777 * NOOPs. blkg_conf_prep() implicitly calls this function. Use this function778 * explicitly if bdev access is needed without resolving the blkcg / policy part779 * of @ctx->input. Returns -errno on error.780 */781int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx)782{783 char *input = ctx->input;784 unsigned int major, minor;785 struct block_device *bdev;786 int key_len;787 788 if (ctx->bdev)789 return 0;790 791 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)792 return -EINVAL;793 794 input += key_len;795 if (!isspace(*input))796 return -EINVAL;797 input = skip_spaces(input);798 799 bdev = blkdev_get_no_open(MKDEV(major, minor));800 if (!bdev)801 return -ENODEV;802 if (bdev_is_partition(bdev)) {803 blkdev_put_no_open(bdev);804 return -ENODEV;805 }806 807 mutex_lock(&bdev->bd_queue->rq_qos_mutex);808 if (!disk_live(bdev->bd_disk)) {809 blkdev_put_no_open(bdev);810 mutex_unlock(&bdev->bd_queue->rq_qos_mutex);811 return -ENODEV;812 }813 814 ctx->body = input;815 ctx->bdev = bdev;816 return 0;817}818 819/**820 * blkg_conf_prep - parse and prepare for per-blkg config update821 * @blkcg: target block cgroup822 * @pol: target policy823 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()824 *825 * Parse per-blkg config update from @ctx->input and initialize @ctx826 * accordingly. On success, @ctx->body points to the part of @ctx->input827 * following MAJ:MIN, @ctx->bdev points to the target block device and828 * @ctx->blkg to the blkg being configured.829 *830 * blkg_conf_open_bdev() may be called on @ctx beforehand. On success, this831 * function returns with queue lock held and must be followed by832 * blkg_conf_exit().833 */834int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,835 struct blkg_conf_ctx *ctx)836 __acquires(&bdev->bd_queue->queue_lock)837{838 struct gendisk *disk;839 struct request_queue *q;840 struct blkcg_gq *blkg;841 int ret;842 843 ret = blkg_conf_open_bdev(ctx);844 if (ret)845 return ret;846 847 disk = ctx->bdev->bd_disk;848 q = disk->queue;849 850 /*851 * blkcg_deactivate_policy() requires queue to be frozen, we can grab852 * q_usage_counter to prevent concurrent with blkcg_deactivate_policy().853 */854 ret = blk_queue_enter(q, 0);855 if (ret)856 goto fail;857 858 spin_lock_irq(&q->queue_lock);859 860 if (!blkcg_policy_enabled(q, pol)) {861 ret = -EOPNOTSUPP;862 goto fail_unlock;863 }864 865 blkg = blkg_lookup(blkcg, q);866 if (blkg)867 goto success;868 869 /*870 * Create blkgs walking down from blkcg_root to @blkcg, so that all871 * non-root blkgs have access to their parents.872 */873 while (true) {874 struct blkcg *pos = blkcg;875 struct blkcg *parent;876 struct blkcg_gq *new_blkg;877 878 parent = blkcg_parent(blkcg);879 while (parent && !blkg_lookup(parent, q)) {880 pos = parent;881 parent = blkcg_parent(parent);882 }883 884 /* Drop locks to do new blkg allocation with GFP_KERNEL. */885 spin_unlock_irq(&q->queue_lock);886 887 new_blkg = blkg_alloc(pos, disk, GFP_KERNEL);888 if (unlikely(!new_blkg)) {889 ret = -ENOMEM;890 goto fail_exit_queue;891 }892 893 if (radix_tree_preload(GFP_KERNEL)) {894 blkg_free(new_blkg);895 ret = -ENOMEM;896 goto fail_exit_queue;897 }898 899 spin_lock_irq(&q->queue_lock);900 901 if (!blkcg_policy_enabled(q, pol)) {902 blkg_free(new_blkg);903 ret = -EOPNOTSUPP;904 goto fail_preloaded;905 }906 907 blkg = blkg_lookup(pos, q);908 if (blkg) {909 blkg_free(new_blkg);910 } else {911 blkg = blkg_create(pos, disk, new_blkg);912 if (IS_ERR(blkg)) {913 ret = PTR_ERR(blkg);914 goto fail_preloaded;915 }916 }917 918 radix_tree_preload_end();919 920 if (pos == blkcg)921 goto success;922 }923success:924 blk_queue_exit(q);925 ctx->blkg = blkg;926 return 0;927 928fail_preloaded:929 radix_tree_preload_end();930fail_unlock:931 spin_unlock_irq(&q->queue_lock);932fail_exit_queue:933 blk_queue_exit(q);934fail:935 /*936 * If queue was bypassing, we should retry. Do so after a937 * short msleep(). It isn't strictly necessary but queue938 * can be bypassing for some time and it's always nice to939 * avoid busy looping.940 */941 if (ret == -EBUSY) {942 msleep(10);943 ret = restart_syscall();944 }945 return ret;946}947EXPORT_SYMBOL_GPL(blkg_conf_prep);948 949/**950 * blkg_conf_exit - clean up per-blkg config update951 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()952 *953 * Clean up after per-blkg config update. This function must be called on all954 * blkg_conf_ctx's initialized with blkg_conf_init().955 */956void blkg_conf_exit(struct blkg_conf_ctx *ctx)957 __releases(&ctx->bdev->bd_queue->queue_lock)958 __releases(&ctx->bdev->bd_queue->rq_qos_mutex)959{960 if (ctx->blkg) {961 spin_unlock_irq(&bdev_get_queue(ctx->bdev)->queue_lock);962 ctx->blkg = NULL;963 }964 965 if (ctx->bdev) {966 mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex);967 blkdev_put_no_open(ctx->bdev);968 ctx->body = NULL;969 ctx->bdev = NULL;970 }971}972EXPORT_SYMBOL_GPL(blkg_conf_exit);973 974static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)975{976 int i;977 978 for (i = 0; i < BLKG_IOSTAT_NR; i++) {979 dst->bytes[i] += src->bytes[i];980 dst->ios[i] += src->ios[i];981 }982}983 984static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)985{986 int i;987 988 for (i = 0; i < BLKG_IOSTAT_NR; i++) {989 dst->bytes[i] -= src->bytes[i];990 dst->ios[i] -= src->ios[i];991 }992}993 994static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,995 struct blkg_iostat *last)996{997 struct blkg_iostat delta;998 unsigned long flags;999 1000 /* propagate percpu delta to global */1001 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);1002 blkg_iostat_set(&delta, cur);1003 blkg_iostat_sub(&delta, last);1004 blkg_iostat_add(&blkg->iostat.cur, &delta);1005 blkg_iostat_add(last, &delta);1006 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);1007}1008 1009static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)1010{1011 struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);1012 struct llist_node *lnode;1013 struct blkg_iostat_set *bisc, *next_bisc;1014 unsigned long flags;1015 1016 rcu_read_lock();1017 1018 lnode = llist_del_all(lhead);1019 if (!lnode)1020 goto out;1021 1022 /*1023 * For covering concurrent parent blkg update from blkg_release().1024 *1025 * When flushing from cgroup, cgroup_rstat_lock is always held, so1026 * this lock won't cause contention most of time.1027 */1028 raw_spin_lock_irqsave(&blkg_stat_lock, flags);1029 1030 /*1031 * Iterate only the iostat_cpu's queued in the lockless list.1032 */1033 llist_for_each_entry_safe(bisc, next_bisc, lnode, lnode) {1034 struct blkcg_gq *blkg = bisc->blkg;1035 struct blkcg_gq *parent = blkg->parent;1036 struct blkg_iostat cur;1037 unsigned int seq;1038 1039 /*1040 * Order assignment of `next_bisc` from `bisc->lnode.next` in1041 * llist_for_each_entry_safe and clearing `bisc->lqueued` for1042 * avoiding to assign `next_bisc` with new next pointer added1043 * in blk_cgroup_bio_start() in case of re-ordering.1044 *1045 * The pair barrier is implied in llist_add() in blk_cgroup_bio_start().1046 */1047 smp_mb();1048 1049 WRITE_ONCE(bisc->lqueued, false);1050 if (bisc == &blkg->iostat)1051 goto propagate_up; /* propagate up to parent only */1052 1053 /* fetch the current per-cpu values */1054 do {1055 seq = u64_stats_fetch_begin(&bisc->sync);1056 blkg_iostat_set(&cur, &bisc->cur);1057 } while (u64_stats_fetch_retry(&bisc->sync, seq));1058 1059 blkcg_iostat_update(blkg, &cur, &bisc->last);1060 1061propagate_up:1062 /* propagate global delta to parent (unless that's root) */1063 if (parent && parent->parent) {1064 blkcg_iostat_update(parent, &blkg->iostat.cur,1065 &blkg->iostat.last);1066 /*1067 * Queue parent->iostat to its blkcg's lockless1068 * list to propagate up to the grandparent if the1069 * iostat hasn't been queued yet.1070 */1071 if (!parent->iostat.lqueued) {1072 struct llist_head *plhead;1073 1074 plhead = per_cpu_ptr(parent->blkcg->lhead, cpu);1075 llist_add(&parent->iostat.lnode, plhead);1076 parent->iostat.lqueued = true;1077 }1078 }1079 }1080 raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);1081out:1082 rcu_read_unlock();1083}1084 1085static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)1086{1087 /* Root-level stats are sourced from system-wide IO stats */1088 if (cgroup_parent(css->cgroup))1089 __blkcg_rstat_flush(css_to_blkcg(css), cpu);1090}1091 1092/*1093 * We source root cgroup stats from the system-wide stats to avoid1094 * tracking the same information twice and incurring overhead when no1095 * cgroups are defined. For that reason, cgroup_rstat_flush in1096 * blkcg_print_stat does not actually fill out the iostat in the root1097 * cgroup's blkcg_gq.1098 *1099 * However, we would like to re-use the printing code between the root and1100 * non-root cgroups to the extent possible. For that reason, we simulate1101 * flushing the root cgroup's stats by explicitly filling in the iostat1102 * with disk level statistics.1103 */1104static void blkcg_fill_root_iostats(void)1105{1106 struct class_dev_iter iter;1107 struct device *dev;1108 1109 class_dev_iter_init(&iter, &block_class, NULL, &disk_type);1110 while ((dev = class_dev_iter_next(&iter))) {1111 struct block_device *bdev = dev_to_bdev(dev);1112 struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg;1113 struct blkg_iostat tmp;1114 int cpu;1115 unsigned long flags;1116 1117 memset(&tmp, 0, sizeof(tmp));1118 for_each_possible_cpu(cpu) {1119 struct disk_stats *cpu_dkstats;1120 1121 cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);1122 tmp.ios[BLKG_IOSTAT_READ] +=1123 cpu_dkstats->ios[STAT_READ];1124 tmp.ios[BLKG_IOSTAT_WRITE] +=1125 cpu_dkstats->ios[STAT_WRITE];1126 tmp.ios[BLKG_IOSTAT_DISCARD] +=1127 cpu_dkstats->ios[STAT_DISCARD];1128 // convert sectors to bytes1129 tmp.bytes[BLKG_IOSTAT_READ] +=1130 cpu_dkstats->sectors[STAT_READ] << 9;1131 tmp.bytes[BLKG_IOSTAT_WRITE] +=1132 cpu_dkstats->sectors[STAT_WRITE] << 9;1133 tmp.bytes[BLKG_IOSTAT_DISCARD] +=1134 cpu_dkstats->sectors[STAT_DISCARD] << 9;1135 }1136 1137 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);1138 blkg_iostat_set(&blkg->iostat.cur, &tmp);1139 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);1140 }1141}1142 1143static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)1144{1145 struct blkg_iostat_set *bis = &blkg->iostat;1146 u64 rbytes, wbytes, rios, wios, dbytes, dios;1147 const char *dname;1148 unsigned seq;1149 int i;1150 1151 if (!blkg->online)1152 return;1153 1154 dname = blkg_dev_name(blkg);1155 if (!dname)1156 return;1157 1158 seq_printf(s, "%s ", dname);1159 1160 do {1161 seq = u64_stats_fetch_begin(&bis->sync);1162 1163 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];1164 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];1165 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];1166 rios = bis->cur.ios[BLKG_IOSTAT_READ];1167 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];1168 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];1169 } while (u64_stats_fetch_retry(&bis->sync, seq));1170 1171 if (rbytes || wbytes || rios || wios) {1172 seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",1173 rbytes, wbytes, rios, wios,1174 dbytes, dios);1175 }1176 1177 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {1178 seq_printf(s, " use_delay=%d delay_nsec=%llu",1179 atomic_read(&blkg->use_delay),1180 atomic64_read(&blkg->delay_nsec));1181 }1182 1183 for (i = 0; i < BLKCG_MAX_POLS; i++) {1184 struct blkcg_policy *pol = blkcg_policy[i];1185 1186 if (!blkg->pd[i] || !pol->pd_stat_fn)1187 continue;1188 1189 pol->pd_stat_fn(blkg->pd[i], s);1190 }1191 1192 seq_puts(s, "\n");1193}1194 1195static int blkcg_print_stat(struct seq_file *sf, void *v)1196{1197 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));1198 struct blkcg_gq *blkg;1199 1200 if (!seq_css(sf)->parent)1201 blkcg_fill_root_iostats();1202 else1203 cgroup_rstat_flush(blkcg->css.cgroup);1204 1205 rcu_read_lock();1206 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {1207 spin_lock_irq(&blkg->q->queue_lock);1208 blkcg_print_one_stat(blkg, sf);1209 spin_unlock_irq(&blkg->q->queue_lock);1210 }1211 rcu_read_unlock();1212 return 0;1213}1214 1215static struct cftype blkcg_files[] = {1216 {1217 .name = "stat",1218 .seq_show = blkcg_print_stat,1219 },1220 { } /* terminate */1221};1222 1223static struct cftype blkcg_legacy_files[] = {1224 {1225 .name = "reset_stats",1226 .write_u64 = blkcg_reset_stats,1227 },1228 { } /* terminate */1229};1230 1231#ifdef CONFIG_CGROUP_WRITEBACK1232struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css)1233{1234 return &css_to_blkcg(css)->cgwb_list;1235}1236#endif1237 1238/*1239 * blkcg destruction is a three-stage process.1240 *1241 * 1. Destruction starts. The blkcg_css_offline() callback is invoked1242 * which offlines writeback. Here we tie the next stage of blkg destruction1243 * to the completion of writeback associated with the blkcg. This lets us1244 * avoid punting potentially large amounts of outstanding writeback to root1245 * while maintaining any ongoing policies. The next stage is triggered when1246 * the nr_cgwbs count goes to zero.1247 *1248 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called1249 * and handles the destruction of blkgs. Here the css reference held by1250 * the blkg is put back eventually allowing blkcg_css_free() to be called.1251 * This work may occur in cgwb_release_workfn() on the cgwb_release1252 * workqueue. Any submitted ios that fail to get the blkg ref will be1253 * punted to the root_blkg.1254 *1255 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.1256 * This finally frees the blkcg.1257 */1258 1259/**1260 * blkcg_destroy_blkgs - responsible for shooting down blkgs1261 * @blkcg: blkcg of interest1262 *1263 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock1264 * is nested inside q lock, this function performs reverse double lock dancing.1265 * Destroying the blkgs releases the reference held on the blkcg's css allowing1266 * blkcg_css_free to eventually be called.1267 *1268 * This is the blkcg counterpart of ioc_release_fn().1269 */1270static void blkcg_destroy_blkgs(struct blkcg *blkcg)1271{1272 might_sleep();1273 1274 spin_lock_irq(&blkcg->lock);1275 1276 while (!hlist_empty(&blkcg->blkg_list)) {1277 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,1278 struct blkcg_gq, blkcg_node);1279 struct request_queue *q = blkg->q;1280 1281 if (need_resched() || !spin_trylock(&q->queue_lock)) {1282 /*1283 * Given that the system can accumulate a huge number1284 * of blkgs in pathological cases, check to see if we1285 * need to rescheduling to avoid softlockup.1286 */1287 spin_unlock_irq(&blkcg->lock);1288 cond_resched();1289 spin_lock_irq(&blkcg->lock);1290 continue;1291 }1292 1293 blkg_destroy(blkg);1294 spin_unlock(&q->queue_lock);1295 }1296 1297 spin_unlock_irq(&blkcg->lock);1298}1299 1300/**1301 * blkcg_pin_online - pin online state1302 * @blkcg_css: blkcg of interest1303 *1304 * While pinned, a blkcg is kept online. This is primarily used to1305 * impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline1306 * while an associated cgwb is still active.1307 */1308void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css)1309{1310 refcount_inc(&css_to_blkcg(blkcg_css)->online_pin);1311}1312 1313/**1314 * blkcg_unpin_online - unpin online state1315 * @blkcg_css: blkcg of interest1316 *1317 * This is primarily used to impedance-match blkg and cgwb lifetimes so1318 * that blkg doesn't go offline while an associated cgwb is still active.1319 * When this count goes to zero, all active cgwbs have finished so the1320 * blkcg can continue destruction by calling blkcg_destroy_blkgs().1321 */1322void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css)1323{1324 struct blkcg *blkcg = css_to_blkcg(blkcg_css);1325 1326 do {1327 if (!refcount_dec_and_test(&blkcg->online_pin))1328 break;1329 blkcg_destroy_blkgs(blkcg);1330 blkcg = blkcg_parent(blkcg);1331 } while (blkcg);1332}1333 1334/**1335 * blkcg_css_offline - cgroup css_offline callback1336 * @css: css of interest1337 *1338 * This function is called when @css is about to go away. Here the cgwbs are1339 * offlined first and only once writeback associated with the blkcg has1340 * finished do we start step 2 (see above).1341 */1342static void blkcg_css_offline(struct cgroup_subsys_state *css)1343{1344 /* this prevents anyone from attaching or migrating to this blkcg */1345 wb_blkcg_offline(css);1346 1347 /* put the base online pin allowing step 2 to be triggered */1348 blkcg_unpin_online(css);1349}1350 1351static void blkcg_css_free(struct cgroup_subsys_state *css)1352{1353 struct blkcg *blkcg = css_to_blkcg(css);1354 int i;1355 1356 mutex_lock(&blkcg_pol_mutex);1357 1358 list_del(&blkcg->all_blkcgs_node);1359 1360 for (i = 0; i < BLKCG_MAX_POLS; i++)1361 if (blkcg->cpd[i])1362 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);1363 1364 mutex_unlock(&blkcg_pol_mutex);1365 1366 free_percpu(blkcg->lhead);1367 kfree(blkcg);1368}1369 1370static struct cgroup_subsys_state *1371blkcg_css_alloc(struct cgroup_subsys_state *parent_css)1372{1373 struct blkcg *blkcg;1374 int i;1375 1376 mutex_lock(&blkcg_pol_mutex);1377 1378 if (!parent_css) {1379 blkcg = &blkcg_root;1380 } else {1381 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);1382 if (!blkcg)1383 goto unlock;1384 }1385 1386 if (init_blkcg_llists(blkcg))1387 goto free_blkcg;1388 1389 for (i = 0; i < BLKCG_MAX_POLS ; i++) {1390 struct blkcg_policy *pol = blkcg_policy[i];1391 struct blkcg_policy_data *cpd;1392 1393 /*1394 * If the policy hasn't been attached yet, wait for it1395 * to be attached before doing anything else. Otherwise,1396 * check if the policy requires any specific per-cgroup1397 * data: if it does, allocate and initialize it.1398 */1399 if (!pol || !pol->cpd_alloc_fn)1400 continue;1401 1402 cpd = pol->cpd_alloc_fn(GFP_KERNEL);1403 if (!cpd)1404 goto free_pd_blkcg;1405 1406 blkcg->cpd[i] = cpd;1407 cpd->blkcg = blkcg;1408 cpd->plid = i;1409 }1410 1411 spin_lock_init(&blkcg->lock);1412 refcount_set(&blkcg->online_pin, 1);1413 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);1414 INIT_HLIST_HEAD(&blkcg->blkg_list);1415#ifdef CONFIG_CGROUP_WRITEBACK1416 INIT_LIST_HEAD(&blkcg->cgwb_list);1417#endif1418 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);1419 1420 mutex_unlock(&blkcg_pol_mutex);1421 return &blkcg->css;1422 1423free_pd_blkcg:1424 for (i--; i >= 0; i--)1425 if (blkcg->cpd[i])1426 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);1427 free_percpu(blkcg->lhead);1428free_blkcg:1429 if (blkcg != &blkcg_root)1430 kfree(blkcg);1431unlock:1432 mutex_unlock(&blkcg_pol_mutex);1433 return ERR_PTR(-ENOMEM);1434}1435 1436static int blkcg_css_online(struct cgroup_subsys_state *css)1437{1438 struct blkcg *parent = blkcg_parent(css_to_blkcg(css));1439 1440 /*1441 * blkcg_pin_online() is used to delay blkcg offline so that blkgs1442 * don't go offline while cgwbs are still active on them. Pin the1443 * parent so that offline always happens towards the root.1444 */1445 if (parent)1446 blkcg_pin_online(&parent->css);1447 return 0;1448}1449 1450void blkg_init_queue(struct request_queue *q)1451{1452 INIT_LIST_HEAD(&q->blkg_list);1453 mutex_init(&q->blkcg_mutex);1454}1455 1456int blkcg_init_disk(struct gendisk *disk)1457{1458 struct request_queue *q = disk->queue;1459 struct blkcg_gq *new_blkg, *blkg;1460 bool preloaded;1461 1462 new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL);1463 if (!new_blkg)1464 return -ENOMEM;1465 1466 preloaded = !radix_tree_preload(GFP_KERNEL);1467 1468 /* Make sure the root blkg exists. */1469 /* spin_lock_irq can serve as RCU read-side critical section. */1470 spin_lock_irq(&q->queue_lock);1471 blkg = blkg_create(&blkcg_root, disk, new_blkg);1472 if (IS_ERR(blkg))1473 goto err_unlock;1474 q->root_blkg = blkg;1475 spin_unlock_irq(&q->queue_lock);1476 1477 if (preloaded)1478 radix_tree_preload_end();1479 1480 return 0;1481 1482err_unlock:1483 spin_unlock_irq(&q->queue_lock);1484 if (preloaded)1485 radix_tree_preload_end();1486 return PTR_ERR(blkg);1487}1488 1489void blkcg_exit_disk(struct gendisk *disk)1490{1491 blkg_destroy_all(disk);1492 blk_throtl_exit(disk);1493}1494 1495static void blkcg_exit(struct task_struct *tsk)1496{1497 if (tsk->throttle_disk)1498 put_disk(tsk->throttle_disk);1499 tsk->throttle_disk = NULL;1500}1501 1502struct cgroup_subsys io_cgrp_subsys = {1503 .css_alloc = blkcg_css_alloc,1504 .css_online = blkcg_css_online,1505 .css_offline = blkcg_css_offline,1506 .css_free = blkcg_css_free,1507 .css_rstat_flush = blkcg_rstat_flush,1508 .dfl_cftypes = blkcg_files,1509 .legacy_cftypes = blkcg_legacy_files,1510 .legacy_name = "blkio",1511 .exit = blkcg_exit,1512#ifdef CONFIG_MEMCG1513 /*1514 * This ensures that, if available, memcg is automatically enabled1515 * together on the default hierarchy so that the owner cgroup can1516 * be retrieved from writeback pages.1517 */1518 .depends_on = 1 << memory_cgrp_id,1519#endif1520};1521EXPORT_SYMBOL_GPL(io_cgrp_subsys);1522 1523/**1524 * blkcg_activate_policy - activate a blkcg policy on a gendisk1525 * @disk: gendisk of interest1526 * @pol: blkcg policy to activate1527 *1528 * Activate @pol on @disk. Requires %GFP_KERNEL context. @disk goes through1529 * bypass mode to populate its blkgs with policy_data for @pol.1530 *1531 * Activation happens with @disk bypassed, so nobody would be accessing blkgs1532 * from IO path. Update of each blkg is protected by both queue and blkcg1533 * locks so that holding either lock and testing blkcg_policy_enabled() is1534 * always enough for dereferencing policy data.1535 *1536 * The caller is responsible for synchronizing [de]activations and policy1537 * [un]registerations. Returns 0 on success, -errno on failure.1538 */1539int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol)1540{1541 struct request_queue *q = disk->queue;1542 struct blkg_policy_data *pd_prealloc = NULL;1543 struct blkcg_gq *blkg, *pinned_blkg = NULL;1544 int ret;1545 1546 if (blkcg_policy_enabled(q, pol))1547 return 0;1548 1549 /*1550 * Policy is allowed to be registered without pd_alloc_fn/pd_free_fn,1551 * for example, ioprio. Such policy will work on blkcg level, not disk1552 * level, and don't need to be activated.1553 */1554 if (WARN_ON_ONCE(!pol->pd_alloc_fn || !pol->pd_free_fn))1555 return -EINVAL;1556 1557 if (queue_is_mq(q))1558 blk_mq_freeze_queue(q);1559retry:1560 spin_lock_irq(&q->queue_lock);1561 1562 /* blkg_list is pushed at the head, reverse walk to initialize parents first */1563 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {1564 struct blkg_policy_data *pd;1565 1566 if (blkg->pd[pol->plid])1567 continue;1568 1569 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */1570 if (blkg == pinned_blkg) {1571 pd = pd_prealloc;1572 pd_prealloc = NULL;1573 } else {1574 pd = pol->pd_alloc_fn(disk, blkg->blkcg,1575 GFP_NOWAIT | __GFP_NOWARN);1576 }1577 1578 if (!pd) {1579 /*1580 * GFP_NOWAIT failed. Free the existing one and1581 * prealloc for @blkg w/ GFP_KERNEL.1582 */1583 if (pinned_blkg)1584 blkg_put(pinned_blkg);1585 blkg_get(blkg);1586 pinned_blkg = blkg;1587 1588 spin_unlock_irq(&q->queue_lock);1589 1590 if (pd_prealloc)1591 pol->pd_free_fn(pd_prealloc);1592 pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,1593 GFP_KERNEL);1594 if (pd_prealloc)1595 goto retry;1596 else1597 goto enomem;1598 }1599 1600 spin_lock(&blkg->blkcg->lock);1601 1602 pd->blkg = blkg;1603 pd->plid = pol->plid;1604 blkg->pd[pol->plid] = pd;1605 1606 if (pol->pd_init_fn)1607 pol->pd_init_fn(pd);1608 1609 if (pol->pd_online_fn)1610 pol->pd_online_fn(pd);1611 pd->online = true;1612 1613 spin_unlock(&blkg->blkcg->lock);1614 }1615 1616 __set_bit(pol->plid, q->blkcg_pols);1617 ret = 0;1618 1619 spin_unlock_irq(&q->queue_lock);1620out:1621 if (queue_is_mq(q))1622 blk_mq_unfreeze_queue(q);1623 if (pinned_blkg)1624 blkg_put(pinned_blkg);1625 if (pd_prealloc)1626 pol->pd_free_fn(pd_prealloc);1627 return ret;1628 1629enomem:1630 /* alloc failed, take down everything */1631 spin_lock_irq(&q->queue_lock);1632 list_for_each_entry(blkg, &q->blkg_list, q_node) {1633 struct blkcg *blkcg = blkg->blkcg;1634 struct blkg_policy_data *pd;1635 1636 spin_lock(&blkcg->lock);1637 pd = blkg->pd[pol->plid];1638 if (pd) {1639 if (pd->online && pol->pd_offline_fn)1640 pol->pd_offline_fn(pd);1641 pd->online = false;1642 pol->pd_free_fn(pd);1643 blkg->pd[pol->plid] = NULL;1644 }1645 spin_unlock(&blkcg->lock);1646 }1647 spin_unlock_irq(&q->queue_lock);1648 ret = -ENOMEM;1649 goto out;1650}1651EXPORT_SYMBOL_GPL(blkcg_activate_policy);1652 1653/**1654 * blkcg_deactivate_policy - deactivate a blkcg policy on a gendisk1655 * @disk: gendisk of interest1656 * @pol: blkcg policy to deactivate1657 *1658 * Deactivate @pol on @disk. Follows the same synchronization rules as1659 * blkcg_activate_policy().1660 */1661void blkcg_deactivate_policy(struct gendisk *disk,1662 const struct blkcg_policy *pol)1663{1664 struct request_queue *q = disk->queue;1665 struct blkcg_gq *blkg;1666 1667 if (!blkcg_policy_enabled(q, pol))1668 return;1669 1670 if (queue_is_mq(q))1671 blk_mq_freeze_queue(q);1672 1673 mutex_lock(&q->blkcg_mutex);1674 spin_lock_irq(&q->queue_lock);1675 1676 __clear_bit(pol->plid, q->blkcg_pols);1677 1678 list_for_each_entry(blkg, &q->blkg_list, q_node) {1679 struct blkcg *blkcg = blkg->blkcg;1680 1681 spin_lock(&blkcg->lock);1682 if (blkg->pd[pol->plid]) {1683 if (blkg->pd[pol->plid]->online && pol->pd_offline_fn)1684 pol->pd_offline_fn(blkg->pd[pol->plid]);1685 pol->pd_free_fn(blkg->pd[pol->plid]);1686 blkg->pd[pol->plid] = NULL;1687 }1688 spin_unlock(&blkcg->lock);1689 }1690 1691 spin_unlock_irq(&q->queue_lock);1692 mutex_unlock(&q->blkcg_mutex);1693 1694 if (queue_is_mq(q))1695 blk_mq_unfreeze_queue(q);1696}1697EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);1698 1699static void blkcg_free_all_cpd(struct blkcg_policy *pol)1700{1701 struct blkcg *blkcg;1702 1703 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {1704 if (blkcg->cpd[pol->plid]) {1705 pol->cpd_free_fn(blkcg->cpd[pol->plid]);1706 blkcg->cpd[pol->plid] = NULL;1707 }1708 }1709}1710 1711/**1712 * blkcg_policy_register - register a blkcg policy1713 * @pol: blkcg policy to register1714 *1715 * Register @pol with blkcg core. Might sleep and @pol may be modified on1716 * successful registration. Returns 0 on success and -errno on failure.1717 */1718int blkcg_policy_register(struct blkcg_policy *pol)1719{1720 struct blkcg *blkcg;1721 int i, ret;1722 1723 mutex_lock(&blkcg_pol_register_mutex);1724 mutex_lock(&blkcg_pol_mutex);1725 1726 /* find an empty slot */1727 ret = -ENOSPC;1728 for (i = 0; i < BLKCG_MAX_POLS; i++)1729 if (!blkcg_policy[i])1730 break;1731 if (i >= BLKCG_MAX_POLS) {1732 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");1733 goto err_unlock;1734 }1735 1736 /*1737 * Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs, and policy1738 * without pd_alloc_fn/pd_free_fn can't be activated.1739 */1740 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||1741 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))1742 goto err_unlock;1743 1744 /* register @pol */1745 pol->plid = i;1746 blkcg_policy[pol->plid] = pol;1747 1748 /* allocate and install cpd's */1749 if (pol->cpd_alloc_fn) {1750 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {1751 struct blkcg_policy_data *cpd;1752 1753 cpd = pol->cpd_alloc_fn(GFP_KERNEL);1754 if (!cpd)1755 goto err_free_cpds;1756 1757 blkcg->cpd[pol->plid] = cpd;1758 cpd->blkcg = blkcg;1759 cpd->plid = pol->plid;1760 }1761 }1762 1763 mutex_unlock(&blkcg_pol_mutex);1764 1765 /* everything is in place, add intf files for the new policy */1766 if (pol->dfl_cftypes)1767 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,1768 pol->dfl_cftypes));1769 if (pol->legacy_cftypes)1770 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,1771 pol->legacy_cftypes));1772 mutex_unlock(&blkcg_pol_register_mutex);1773 return 0;1774 1775err_free_cpds:1776 if (pol->cpd_free_fn)1777 blkcg_free_all_cpd(pol);1778 1779 blkcg_policy[pol->plid] = NULL;1780err_unlock:1781 mutex_unlock(&blkcg_pol_mutex);1782 mutex_unlock(&blkcg_pol_register_mutex);1783 return ret;1784}1785EXPORT_SYMBOL_GPL(blkcg_policy_register);1786 1787/**1788 * blkcg_policy_unregister - unregister a blkcg policy1789 * @pol: blkcg policy to unregister1790 *1791 * Undo blkcg_policy_register(@pol). Might sleep.1792 */1793void blkcg_policy_unregister(struct blkcg_policy *pol)1794{1795 mutex_lock(&blkcg_pol_register_mutex);1796 1797 if (WARN_ON(blkcg_policy[pol->plid] != pol))1798 goto out_unlock;1799 1800 /* kill the intf files first */1801 if (pol->dfl_cftypes)1802 cgroup_rm_cftypes(pol->dfl_cftypes);1803 if (pol->legacy_cftypes)1804 cgroup_rm_cftypes(pol->legacy_cftypes);1805 1806 /* remove cpds and unregister */1807 mutex_lock(&blkcg_pol_mutex);1808 1809 if (pol->cpd_free_fn)1810 blkcg_free_all_cpd(pol);1811 1812 blkcg_policy[pol->plid] = NULL;1813 1814 mutex_unlock(&blkcg_pol_mutex);1815out_unlock:1816 mutex_unlock(&blkcg_pol_register_mutex);1817}1818EXPORT_SYMBOL_GPL(blkcg_policy_unregister);1819 1820/*1821 * Scale the accumulated delay based on how long it has been since we updated1822 * the delay. We only call this when we are adding delay, in case it's been a1823 * while since we added delay, and when we are checking to see if we need to1824 * delay a task, to account for any delays that may have occurred.1825 */1826static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)1827{1828 u64 old = atomic64_read(&blkg->delay_start);1829 1830 /* negative use_delay means no scaling, see blkcg_set_delay() */1831 if (atomic_read(&blkg->use_delay) < 0)1832 return;1833 1834 /*1835 * We only want to scale down every second. The idea here is that we1836 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain1837 * time window. We only want to throttle tasks for recent delay that1838 * has occurred, in 1 second time windows since that's the maximum1839 * things can be throttled. We save the current delay window in1840 * blkg->last_delay so we know what amount is still left to be charged1841 * to the blkg from this point onward. blkg->last_use keeps track of1842 * the use_delay counter. The idea is if we're unthrottling the blkg we1843 * are ok with whatever is happening now, and we can take away more of1844 * the accumulated delay as we've already throttled enough that1845 * everybody is happy with their IO latencies.1846 */1847 if (time_before64(old + NSEC_PER_SEC, now) &&1848 atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) {1849 u64 cur = atomic64_read(&blkg->delay_nsec);1850 u64 sub = min_t(u64, blkg->last_delay, now - old);1851 int cur_use = atomic_read(&blkg->use_delay);1852 1853 /*1854 * We've been unthrottled, subtract a larger chunk of our1855 * accumulated delay.1856 */1857 if (cur_use < blkg->last_use)1858 sub = max_t(u64, sub, blkg->last_delay >> 1);1859 1860 /*1861 * This shouldn't happen, but handle it anyway. Our delay_nsec1862 * should only ever be growing except here where we subtract out1863 * min(last_delay, 1 second), but lord knows bugs happen and I'd1864 * rather not end up with negative numbers.1865 */1866 if (unlikely(cur < sub)) {1867 atomic64_set(&blkg->delay_nsec, 0);1868 blkg->last_delay = 0;1869 } else {1870 atomic64_sub(sub, &blkg->delay_nsec);1871 blkg->last_delay = cur - sub;1872 }1873 blkg->last_use = cur_use;1874 }1875}1876 1877/*1878 * This is called when we want to actually walk up the hierarchy and check to1879 * see if we need to throttle, and then actually throttle if there is some1880 * accumulated delay. This should only be called upon return to user space so1881 * we're not holding some lock that would induce a priority inversion.1882 */1883static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)1884{1885 unsigned long pflags;1886 bool clamp;1887 u64 now = blk_time_get_ns();1888 u64 exp;1889 u64 delay_nsec = 0;1890 int tok;1891 1892 while (blkg->parent) {1893 int use_delay = atomic_read(&blkg->use_delay);1894 1895 if (use_delay) {1896 u64 this_delay;1897 1898 blkcg_scale_delay(blkg, now);1899 this_delay = atomic64_read(&blkg->delay_nsec);1900 if (this_delay > delay_nsec) {1901 delay_nsec = this_delay;1902 clamp = use_delay > 0;1903 }1904 }1905 blkg = blkg->parent;1906 }1907 1908 if (!delay_nsec)1909 return;1910 1911 /*1912 * Let's not sleep for all eternity if we've amassed a huge delay.1913 * Swapping or metadata IO can accumulate 10's of seconds worth of1914 * delay, and we want userspace to be able to do _something_ so cap the1915 * delays at 0.25s. If there's 10's of seconds worth of delay then the1916 * tasks will be delayed for 0.25 second for every syscall. If1917 * blkcg_set_delay() was used as indicated by negative use_delay, the1918 * caller is responsible for regulating the range.1919 */1920 if (clamp)1921 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);1922 1923 if (use_memdelay)1924 psi_memstall_enter(&pflags);1925 1926 exp = ktime_add_ns(now, delay_nsec);1927 tok = io_schedule_prepare();1928 do {1929 __set_current_state(TASK_KILLABLE);1930 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))1931 break;1932 } while (!fatal_signal_pending(current));1933 io_schedule_finish(tok);1934 1935 if (use_memdelay)1936 psi_memstall_leave(&pflags);1937}1938 1939/**1940 * blkcg_maybe_throttle_current - throttle the current task if it has been marked1941 *1942 * This is only called if we've been marked with set_notify_resume(). Obviously1943 * we can be set_notify_resume() for reasons other than blkcg throttling, so we1944 * check to see if current->throttle_disk is set and if not this doesn't do1945 * anything. This should only ever be called by the resume code, it's not meant1946 * to be called by people willy-nilly as it will actually do the work to1947 * throttle the task if it is setup for throttling.1948 */1949void blkcg_maybe_throttle_current(void)1950{1951 struct gendisk *disk = current->throttle_disk;1952 struct blkcg *blkcg;1953 struct blkcg_gq *blkg;1954 bool use_memdelay = current->use_memdelay;1955 1956 if (!disk)1957 return;1958 1959 current->throttle_disk = NULL;1960 current->use_memdelay = false;1961 1962 rcu_read_lock();1963 blkcg = css_to_blkcg(blkcg_css());1964 if (!blkcg)1965 goto out;1966 blkg = blkg_lookup(blkcg, disk->queue);1967 if (!blkg)1968 goto out;1969 if (!blkg_tryget(blkg))1970 goto out;1971 rcu_read_unlock();1972 1973 blkcg_maybe_throttle_blkg(blkg, use_memdelay);1974 blkg_put(blkg);1975 put_disk(disk);1976 return;1977out:1978 rcu_read_unlock();1979}1980 1981/**1982 * blkcg_schedule_throttle - this task needs to check for throttling1983 * @disk: disk to throttle1984 * @use_memdelay: do we charge this to memory delay for PSI1985 *1986 * This is called by the IO controller when we know there's delay accumulated1987 * for the blkg for this task. We do not pass the blkg because there are places1988 * we call this that may not have that information, the swapping code for1989 * instance will only have a block_device at that point. This set's the1990 * notify_resume for the task to check and see if it requires throttling before1991 * returning to user space.1992 *1993 * We will only schedule once per syscall. You can call this over and over1994 * again and it will only do the check once upon return to user space, and only1995 * throttle once. If the task needs to be throttled again it'll need to be1996 * re-set at the next time we see the task.1997 */1998void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay)1999{2000 if (unlikely(current->flags & PF_KTHREAD))2001 return;2002 2003 if (current->throttle_disk != disk) {2004 if (test_bit(GD_DEAD, &disk->state))2005 return;2006 get_device(disk_to_dev(disk));2007 2008 if (current->throttle_disk)2009 put_disk(current->throttle_disk);2010 current->throttle_disk = disk;2011 }2012 2013 if (use_memdelay)2014 current->use_memdelay = use_memdelay;2015 set_notify_resume(current);2016}2017 2018/**2019 * blkcg_add_delay - add delay to this blkg2020 * @blkg: blkg of interest2021 * @now: the current time in nanoseconds2022 * @delta: how many nanoseconds of delay to add2023 *2024 * Charge @delta to the blkg's current delay accumulation. This is used to2025 * throttle tasks if an IO controller thinks we need more throttling.2026 */2027void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)2028{2029 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))2030 return;2031 blkcg_scale_delay(blkg, now);2032 atomic64_add(delta, &blkg->delay_nsec);2033}2034 2035/**2036 * blkg_tryget_closest - try and get a blkg ref on the closet blkg2037 * @bio: target bio2038 * @css: target css2039 *2040 * As the failure mode here is to walk up the blkg tree, this ensure that the2041 * blkg->parent pointers are always valid. This returns the blkg that it ended2042 * up taking a reference on or %NULL if no reference was taken.2043 */2044static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,2045 struct cgroup_subsys_state *css)2046{2047 struct blkcg_gq *blkg, *ret_blkg = NULL;2048 2049 rcu_read_lock();2050 blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk);2051 while (blkg) {2052 if (blkg_tryget(blkg)) {2053 ret_blkg = blkg;2054 break;2055 }2056 blkg = blkg->parent;2057 }2058 rcu_read_unlock();2059 2060 return ret_blkg;2061}2062 2063/**2064 * bio_associate_blkg_from_css - associate a bio with a specified css2065 * @bio: target bio2066 * @css: target css2067 *2068 * Associate @bio with the blkg found by combining the css's blkg and the2069 * request_queue of the @bio. An association failure is handled by walking up2070 * the blkg tree. Therefore, the blkg associated can be anything between @blkg2071 * and q->root_blkg. This situation only happens when a cgroup is dying and2072 * then the remaining bios will spill to the closest alive blkg.2073 *2074 * A reference will be taken on the blkg and will be released when @bio is2075 * freed.2076 */2077void bio_associate_blkg_from_css(struct bio *bio,2078 struct cgroup_subsys_state *css)2079{2080 if (bio->bi_blkg)2081 blkg_put(bio->bi_blkg);2082 2083 if (css && css->parent) {2084 bio->bi_blkg = blkg_tryget_closest(bio, css);2085 } else {2086 blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);2087 bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;2088 }2089}2090EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);2091 2092/**2093 * bio_associate_blkg - associate a bio with a blkg2094 * @bio: target bio2095 *2096 * Associate @bio with the blkg found from the bio's css and request_queue.2097 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is2098 * already associated, the css is reused and association redone as the2099 * request_queue may have changed.2100 */2101void bio_associate_blkg(struct bio *bio)2102{2103 struct cgroup_subsys_state *css;2104 2105 if (blk_op_is_passthrough(bio->bi_opf))2106 return;2107 2108 rcu_read_lock();2109 2110 if (bio->bi_blkg)2111 css = bio_blkcg_css(bio);2112 else2113 css = blkcg_css();2114 2115 bio_associate_blkg_from_css(bio, css);2116 2117 rcu_read_unlock();2118}2119EXPORT_SYMBOL_GPL(bio_associate_blkg);2120 2121/**2122 * bio_clone_blkg_association - clone blkg association from src to dst bio2123 * @dst: destination bio2124 * @src: source bio2125 */2126void bio_clone_blkg_association(struct bio *dst, struct bio *src)2127{2128 if (src->bi_blkg)2129 bio_associate_blkg_from_css(dst, bio_blkcg_css(src));2130}2131EXPORT_SYMBOL_GPL(bio_clone_blkg_association);2132 2133static int blk_cgroup_io_type(struct bio *bio)2134{2135 if (op_is_discard(bio->bi_opf))2136 return BLKG_IOSTAT_DISCARD;2137 if (op_is_write(bio->bi_opf))2138 return BLKG_IOSTAT_WRITE;2139 return BLKG_IOSTAT_READ;2140}2141 2142void blk_cgroup_bio_start(struct bio *bio)2143{2144 struct blkcg *blkcg = bio->bi_blkg->blkcg;2145 int rwd = blk_cgroup_io_type(bio), cpu;2146 struct blkg_iostat_set *bis;2147 unsigned long flags;2148 2149 if (!cgroup_subsys_on_dfl(io_cgrp_subsys))2150 return;2151 2152 /* Root-level stats are sourced from system-wide IO stats */2153 if (!cgroup_parent(blkcg->css.cgroup))2154 return;2155 2156 cpu = get_cpu();2157 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);2158 flags = u64_stats_update_begin_irqsave(&bis->sync);2159 2160 /*2161 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split2162 * bio and we would have already accounted for the size of the bio.2163 */2164 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {2165 bio_set_flag(bio, BIO_CGROUP_ACCT);2166 bis->cur.bytes[rwd] += bio->bi_iter.bi_size;2167 }2168 bis->cur.ios[rwd]++;2169 2170 /*2171 * If the iostat_cpu isn't in a lockless list, put it into the2172 * list to indicate that a stat update is pending.2173 */2174 if (!READ_ONCE(bis->lqueued)) {2175 struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);2176 2177 llist_add(&bis->lnode, lhead);2178 WRITE_ONCE(bis->lqueued, true);2179 }2180 2181 u64_stats_update_end_irqrestore(&bis->sync, flags);2182 cgroup_rstat_updated(blkcg->css.cgroup, cpu);2183 put_cpu();2184}2185 2186bool blk_cgroup_congested(void)2187{2188 struct blkcg *blkcg;2189 bool ret = false;2190 2191 rcu_read_lock();2192 for (blkcg = css_to_blkcg(blkcg_css()); blkcg;2193 blkcg = blkcg_parent(blkcg)) {2194 if (atomic_read(&blkcg->congestion_count)) {2195 ret = true;2196 break;2197 }2198 }2199 rcu_read_unlock();2200 return ret;2201}2202 2203module_param(blkcg_debug_stats, bool, 0644);2204MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");2205