204 lines · c
1#ifndef BLK_THROTTLE_H2#define BLK_THROTTLE_H3 4#include "blk-cgroup-rwstat.h"5 6/*7 * To implement hierarchical throttling, throtl_grps form a tree and bios8 * are dispatched upwards level by level until they reach the top and get9 * issued. When dispatching bios from the children and local group at each10 * level, if the bios are dispatched into a single bio_list, there's a risk11 * of a local or child group which can queue many bios at once filling up12 * the list starving others.13 *14 * To avoid such starvation, dispatched bios are queued separately15 * according to where they came from. When they are again dispatched to16 * the parent, they're popped in round-robin order so that no single source17 * hogs the dispatch window.18 *19 * throtl_qnode is used to keep the queued bios separated by their sources.20 * Bios are queued to throtl_qnode which in turn is queued to21 * throtl_service_queue and then dispatched in round-robin order.22 *23 * It's also used to track the reference counts on blkg's. A qnode always24 * belongs to a throtl_grp and gets queued on itself or the parent, so25 * incrementing the reference of the associated throtl_grp when a qnode is26 * queued and decrementing when dequeued is enough to keep the whole blkg27 * tree pinned while bios are in flight.28 */29struct throtl_qnode {30 struct list_head node; /* service_queue->queued[] */31 struct bio_list bios; /* queued bios */32 struct throtl_grp *tg; /* tg this qnode belongs to */33};34 35struct throtl_service_queue {36 struct throtl_service_queue *parent_sq; /* the parent service_queue */37 38 /*39 * Bios queued directly to this service_queue or dispatched from40 * children throtl_grp's.41 */42 struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */43 unsigned int nr_queued[2]; /* number of queued bios */44 45 /*46 * RB tree of active children throtl_grp's, which are sorted by47 * their ->disptime.48 */49 struct rb_root_cached pending_tree; /* RB tree of active tgs */50 unsigned int nr_pending; /* # queued in the tree */51 unsigned long first_pending_disptime; /* disptime of the first tg */52 struct timer_list pending_timer; /* fires on first_pending_disptime */53};54 55enum tg_state_flags {56 THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */57 THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */58 THROTL_TG_CANCELING = 1 << 2, /* starts to cancel bio */59};60 61struct throtl_grp {62 /* must be the first member */63 struct blkg_policy_data pd;64 65 /* active throtl group service_queue member */66 struct rb_node rb_node;67 68 /* throtl_data this group belongs to */69 struct throtl_data *td;70 71 /* this group's service queue */72 struct throtl_service_queue service_queue;73 74 /*75 * qnode_on_self is used when bios are directly queued to this76 * throtl_grp so that local bios compete fairly with bios77 * dispatched from children. qnode_on_parent is used when bios are78 * dispatched from this throtl_grp into its parent and will compete79 * with the sibling qnode_on_parents and the parent's80 * qnode_on_self.81 */82 struct throtl_qnode qnode_on_self[2];83 struct throtl_qnode qnode_on_parent[2];84 85 /*86 * Dispatch time in jiffies. This is the estimated time when group87 * will unthrottle and is ready to dispatch more bio. It is used as88 * key to sort active groups in service tree.89 */90 unsigned long disptime;91 92 unsigned int flags;93 94 /* are there any throtl rules between this group and td? */95 bool has_rules_bps[2];96 bool has_rules_iops[2];97 98 /* bytes per second rate limits */99 uint64_t bps[2];100 101 /* IOPS limits */102 unsigned int iops[2];103 104 /* Number of bytes dispatched in current slice */105 uint64_t bytes_disp[2];106 /* Number of bio's dispatched in current slice */107 unsigned int io_disp[2];108 109 uint64_t last_bytes_disp[2];110 unsigned int last_io_disp[2];111 112 /*113 * The following two fields are updated when new configuration is114 * submitted while some bios are still throttled, they record how many115 * bytes/ios are waited already in previous configuration, and they will116 * be used to calculate wait time under new configuration.117 */118 long long carryover_bytes[2];119 int carryover_ios[2];120 121 unsigned long last_check_time;122 123 /* When did we start a new slice */124 unsigned long slice_start[2];125 unsigned long slice_end[2];126 127 struct blkg_rwstat stat_bytes;128 struct blkg_rwstat stat_ios;129};130 131extern struct blkcg_policy blkcg_policy_throtl;132 133static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)134{135 return pd ? container_of(pd, struct throtl_grp, pd) : NULL;136}137 138static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)139{140 return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));141}142 143/*144 * Internal throttling interface145 */146#ifndef CONFIG_BLK_DEV_THROTTLING147static inline void blk_throtl_exit(struct gendisk *disk) { }148static inline bool blk_throtl_bio(struct bio *bio) { return false; }149static inline void blk_throtl_cancel_bios(struct gendisk *disk) { }150#else /* CONFIG_BLK_DEV_THROTTLING */151void blk_throtl_exit(struct gendisk *disk);152bool __blk_throtl_bio(struct bio *bio);153void blk_throtl_cancel_bios(struct gendisk *disk);154 155static inline bool blk_throtl_activated(struct request_queue *q)156{157 return q->td != NULL;158}159 160static inline bool blk_should_throtl(struct bio *bio)161{162 struct throtl_grp *tg;163 int rw = bio_data_dir(bio);164 165 /*166 * This is called under bio_queue_enter(), and it's synchronized with167 * the activation of blk-throtl, which is protected by168 * blk_mq_freeze_queue().169 */170 if (!blk_throtl_activated(bio->bi_bdev->bd_queue))171 return false;172 173 tg = blkg_to_tg(bio->bi_blkg);174 if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {175 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {176 bio_set_flag(bio, BIO_CGROUP_ACCT);177 blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,178 bio->bi_iter.bi_size);179 }180 blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);181 }182 183 /* iops limit is always counted */184 if (tg->has_rules_iops[rw])185 return true;186 187 if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED))188 return true;189 190 return false;191}192 193static inline bool blk_throtl_bio(struct bio *bio)194{195 196 if (!blk_should_throtl(bio))197 return false;198 199 return __blk_throtl_bio(bio);200}201#endif /* CONFIG_BLK_DEV_THROTTLING */202 203#endif204