300 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#ifndef __LINUX_BACKING_DEV_DEFS_H3#define __LINUX_BACKING_DEV_DEFS_H4 5#include <linux/list.h>6#include <linux/radix-tree.h>7#include <linux/rbtree.h>8#include <linux/spinlock.h>9#include <linux/percpu_counter.h>10#include <linux/percpu-refcount.h>11#include <linux/flex_proportions.h>12#include <linux/timer.h>13#include <linux/workqueue.h>14#include <linux/kref.h>15#include <linux/refcount.h>16 17struct page;18struct device;19struct dentry;20 21/*22 * Bits in bdi_writeback.state23 */24enum wb_state {25 WB_registered, /* bdi_register() was done */26 WB_writeback_running, /* Writeback is in progress */27 WB_has_dirty_io, /* Dirty inodes on ->b_{dirty|io|more_io} */28 WB_start_all, /* nr_pages == 0 (all) work pending */29};30 31enum wb_stat_item {32 WB_RECLAIMABLE,33 WB_WRITEBACK,34 WB_DIRTIED,35 WB_WRITTEN,36 NR_WB_STAT_ITEMS37};38 39#define WB_STAT_BATCH (8*(1+ilog2(nr_cpu_ids)))40 41/*42 * why some writeback work was initiated43 */44enum wb_reason {45 WB_REASON_BACKGROUND,46 WB_REASON_VMSCAN,47 WB_REASON_SYNC,48 WB_REASON_PERIODIC,49 WB_REASON_LAPTOP_TIMER,50 WB_REASON_FS_FREE_SPACE,51 /*52 * There is no bdi forker thread any more and works are done53 * by emergency worker, however, this is TPs userland visible54 * and we'll be exposing exactly the same information,55 * so it has a mismatch name.56 */57 WB_REASON_FORKER_THREAD,58 WB_REASON_FOREIGN_FLUSH,59 60 WB_REASON_MAX,61};62 63struct wb_completion {64 atomic_t cnt;65 wait_queue_head_t *waitq;66};67 68#define __WB_COMPLETION_INIT(_waitq) \69 (struct wb_completion){ .cnt = ATOMIC_INIT(1), .waitq = (_waitq) }70 71/*72 * If one wants to wait for one or more wb_writeback_works, each work's73 * ->done should be set to a wb_completion defined using the following74 * macro. Once all work items are issued with wb_queue_work(), the caller75 * can wait for the completion of all using wb_wait_for_completion(). Work76 * items which are waited upon aren't freed automatically on completion.77 */78#define WB_COMPLETION_INIT(bdi) __WB_COMPLETION_INIT(&(bdi)->wb_waitq)79 80#define DEFINE_WB_COMPLETION(cmpl, bdi) \81 struct wb_completion cmpl = WB_COMPLETION_INIT(bdi)82 83/*84 * Each wb (bdi_writeback) can perform writeback operations, is measured85 * and throttled, independently. Without cgroup writeback, each bdi86 * (bdi_writeback) is served by its embedded bdi->wb.87 *88 * On the default hierarchy, blkcg implicitly enables memcg. This allows89 * using memcg's page ownership for attributing writeback IOs, and every90 * memcg - blkcg combination can be served by its own wb by assigning a91 * dedicated wb to each memcg, which enables isolation across different92 * cgroups and propagation of IO back pressure down from the IO layer upto93 * the tasks which are generating the dirty pages to be written back.94 *95 * A cgroup wb is indexed on its bdi by the ID of the associated memcg,96 * refcounted with the number of inodes attached to it, and pins the memcg97 * and the corresponding blkcg. As the corresponding blkcg for a memcg may98 * change as blkcg is disabled and enabled higher up in the hierarchy, a wb99 * is tested for blkcg after lookup and removed from index on mismatch so100 * that a new wb for the combination can be created.101 *102 * Each bdi_writeback that is not embedded into the backing_dev_info must hold103 * a reference to the parent backing_dev_info. See cgwb_create() for details.104 */105struct bdi_writeback {106 struct backing_dev_info *bdi; /* our parent bdi */107 108 unsigned long state; /* Always use atomic bitops on this */109 unsigned long last_old_flush; /* last old data flush */110 111 struct list_head b_dirty; /* dirty inodes */112 struct list_head b_io; /* parked for writeback */113 struct list_head b_more_io; /* parked for more writeback */114 struct list_head b_dirty_time; /* time stamps are dirty */115 spinlock_t list_lock; /* protects the b_* lists */116 117 atomic_t writeback_inodes; /* number of inodes under writeback */118 struct percpu_counter stat[NR_WB_STAT_ITEMS];119 120 unsigned long bw_time_stamp; /* last time write bw is updated */121 unsigned long dirtied_stamp;122 unsigned long written_stamp; /* pages written at bw_time_stamp */123 unsigned long write_bandwidth; /* the estimated write bandwidth */124 unsigned long avg_write_bandwidth; /* further smoothed write bw, > 0 */125 126 /*127 * The base dirty throttle rate, re-calculated on every 200ms.128 * All the bdi tasks' dirty rate will be curbed under it.129 * @dirty_ratelimit tracks the estimated @balanced_dirty_ratelimit130 * in small steps and is much more smooth/stable than the latter.131 */132 unsigned long dirty_ratelimit;133 unsigned long balanced_dirty_ratelimit;134 135 struct fprop_local_percpu completions;136 int dirty_exceeded;137 enum wb_reason start_all_reason;138 139 spinlock_t work_lock; /* protects work_list & dwork scheduling */140 struct list_head work_list;141 struct delayed_work dwork; /* work item used for writeback */142 struct delayed_work bw_dwork; /* work item used for bandwidth estimate */143 144 struct list_head bdi_node; /* anchored at bdi->wb_list */145 146#ifdef CONFIG_CGROUP_WRITEBACK147 struct percpu_ref refcnt; /* used only for !root wb's */148 struct fprop_local_percpu memcg_completions;149 struct cgroup_subsys_state *memcg_css; /* the associated memcg */150 struct cgroup_subsys_state *blkcg_css; /* and blkcg */151 struct list_head memcg_node; /* anchored at memcg->cgwb_list */152 struct list_head blkcg_node; /* anchored at blkcg->cgwb_list */153 struct list_head b_attached; /* attached inodes, protected by list_lock */154 struct list_head offline_node; /* anchored at offline_cgwbs */155 156 union {157 struct work_struct release_work;158 struct rcu_head rcu;159 };160#endif161};162 163struct backing_dev_info {164 u64 id;165 struct rb_node rb_node; /* keyed by ->id */166 struct list_head bdi_list;167 unsigned long ra_pages; /* max readahead in PAGE_SIZE units */168 unsigned long io_pages; /* max allowed IO size */169 170 struct kref refcnt; /* Reference counter for the structure */171 unsigned int capabilities; /* Device capabilities */172 unsigned int min_ratio;173 unsigned int max_ratio, max_prop_frac;174 175 /*176 * Sum of avg_write_bw of wbs with dirty inodes. > 0 if there are177 * any dirty wbs, which is depended upon by bdi_has_dirty().178 */179 atomic_long_t tot_write_bandwidth;180 /*181 * Jiffies when last process was dirty throttled on this bdi. Used by182 * blk-wbt.183 */184 unsigned long last_bdp_sleep;185 186 struct bdi_writeback wb; /* the root writeback info for this bdi */187 struct list_head wb_list; /* list of all wbs */188#ifdef CONFIG_CGROUP_WRITEBACK189 struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */190 struct mutex cgwb_release_mutex; /* protect shutdown of wb structs */191 struct rw_semaphore wb_switch_rwsem; /* no cgwb switch while syncing */192#endif193 wait_queue_head_t wb_waitq;194 195 struct device *dev;196 char dev_name[64];197 struct device *owner;198 199 struct timer_list laptop_mode_wb_timer;200 201#ifdef CONFIG_DEBUG_FS202 struct dentry *debug_dir;203#endif204};205 206struct wb_lock_cookie {207 bool locked;208 unsigned long flags;209};210 211#ifdef CONFIG_CGROUP_WRITEBACK212 213/**214 * wb_tryget - try to increment a wb's refcount215 * @wb: bdi_writeback to get216 */217static inline bool wb_tryget(struct bdi_writeback *wb)218{219 if (wb != &wb->bdi->wb)220 return percpu_ref_tryget(&wb->refcnt);221 return true;222}223 224/**225 * wb_get - increment a wb's refcount226 * @wb: bdi_writeback to get227 */228static inline void wb_get(struct bdi_writeback *wb)229{230 if (wb != &wb->bdi->wb)231 percpu_ref_get(&wb->refcnt);232}233 234/**235 * wb_put - decrement a wb's refcount236 * @wb: bdi_writeback to put237 * @nr: number of references to put238 */239static inline void wb_put_many(struct bdi_writeback *wb, unsigned long nr)240{241 if (WARN_ON_ONCE(!wb->bdi)) {242 /*243 * A driver bug might cause a file to be removed before bdi was244 * initialized.245 */246 return;247 }248 249 if (wb != &wb->bdi->wb)250 percpu_ref_put_many(&wb->refcnt, nr);251}252 253/**254 * wb_put - decrement a wb's refcount255 * @wb: bdi_writeback to put256 */257static inline void wb_put(struct bdi_writeback *wb)258{259 wb_put_many(wb, 1);260}261 262/**263 * wb_dying - is a wb dying?264 * @wb: bdi_writeback of interest265 *266 * Returns whether @wb is unlinked and being drained.267 */268static inline bool wb_dying(struct bdi_writeback *wb)269{270 return percpu_ref_is_dying(&wb->refcnt);271}272 273#else /* CONFIG_CGROUP_WRITEBACK */274 275static inline bool wb_tryget(struct bdi_writeback *wb)276{277 return true;278}279 280static inline void wb_get(struct bdi_writeback *wb)281{282}283 284static inline void wb_put(struct bdi_writeback *wb)285{286}287 288static inline void wb_put_many(struct bdi_writeback *wb, unsigned long nr)289{290}291 292static inline bool wb_dying(struct bdi_writeback *wb)293{294 return false;295}296 297#endif /* CONFIG_CGROUP_WRITEBACK */298 299#endif /* __LINUX_BACKING_DEV_DEFS_H */300