brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/*3 * fs/fs-writeback.c4 *5 * Copyright (C) 2002, Linus Torvalds.6 *7 * Contains all the functions related to writing back and waiting8 * upon dirty inodes against superblocks, and writing back dirty9 * pages against inodes.  ie: data writeback.  Writeout of the10 * inode itself is not handled here.11 *12 * 10Apr2002	Andrew Morton13 *		Split out of fs/inode.c14 *		Additions for address_space-based writeback15 */16 17#include <linux/kernel.h>18#include <linux/export.h>19#include <linux/spinlock.h>20#include <linux/slab.h>21#include <linux/sched.h>22#include <linux/fs.h>23#include <linux/mm.h>24#include <linux/pagemap.h>25#include <linux/kthread.h>26#include <linux/writeback.h>27#include <linux/blkdev.h>28#include <linux/backing-dev.h>29#include <linux/tracepoint.h>30#include <linux/device.h>31#include <linux/memcontrol.h>32#include "internal.h"33 34/*35 * 4MB minimal write chunk size36 */37#define MIN_WRITEBACK_PAGES	(4096UL >> (PAGE_SHIFT - 10))38 39/*40 * Passed into wb_writeback(), essentially a subset of writeback_control41 */42struct wb_writeback_work {43	long nr_pages;44	struct super_block *sb;45	enum writeback_sync_modes sync_mode;46	unsigned int tagged_writepages:1;47	unsigned int for_kupdate:1;48	unsigned int range_cyclic:1;49	unsigned int for_background:1;50	unsigned int for_sync:1;	/* sync(2) WB_SYNC_ALL writeback */51	unsigned int auto_free:1;	/* free on completion */52	enum wb_reason reason;		/* why was writeback initiated? */53 54	struct list_head list;		/* pending work list */55	struct wb_completion *done;	/* set if the caller waits */56};57 58/*59 * If an inode is constantly having its pages dirtied, but then the60 * updates stop dirtytime_expire_interval seconds in the past, it's61 * possible for the worst case time between when an inode has its62 * timestamps updated and when they finally get written out to be two63 * dirtytime_expire_intervals.  We set the default to 12 hours (in64 * seconds), which means most of the time inodes will have their65 * timestamps written to disk after 12 hours, but in the worst case a66 * few inodes might not their timestamps updated for 24 hours.67 */68unsigned int dirtytime_expire_interval = 12 * 60 * 60;69 70static inline struct inode *wb_inode(struct list_head *head)71{72	return list_entry(head, struct inode, i_io_list);73}74 75/*76 * Include the creation of the trace points after defining the77 * wb_writeback_work structure and inline functions so that the definition78 * remains local to this file.79 */80#define CREATE_TRACE_POINTS81#include <trace/events/writeback.h>82 83EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage);84 85static bool wb_io_lists_populated(struct bdi_writeback *wb)86{87	if (wb_has_dirty_io(wb)) {88		return false;89	} else {90		set_bit(WB_has_dirty_io, &wb->state);91		WARN_ON_ONCE(!wb->avg_write_bandwidth);92		atomic_long_add(wb->avg_write_bandwidth,93				&wb->bdi->tot_write_bandwidth);94		return true;95	}96}97 98static void wb_io_lists_depopulated(struct bdi_writeback *wb)99{100	if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&101	    list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {102		clear_bit(WB_has_dirty_io, &wb->state);103		WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,104					&wb->bdi->tot_write_bandwidth) < 0);105	}106}107 108/**109 * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list110 * @inode: inode to be moved111 * @wb: target bdi_writeback112 * @head: one of @wb->b_{dirty|io|more_io|dirty_time}113 *114 * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io.115 * Returns %true if @inode is the first occupant of the !dirty_time IO116 * lists; otherwise, %false.117 */118static bool inode_io_list_move_locked(struct inode *inode,119				      struct bdi_writeback *wb,120				      struct list_head *head)121{122	assert_spin_locked(&wb->list_lock);123	assert_spin_locked(&inode->i_lock);124	WARN_ON_ONCE(inode->i_state & I_FREEING);125 126	list_move(&inode->i_io_list, head);127 128	/* dirty_time doesn't count as dirty_io until expiration */129	if (head != &wb->b_dirty_time)130		return wb_io_lists_populated(wb);131 132	wb_io_lists_depopulated(wb);133	return false;134}135 136static HWJS_SUSPENDS void wb_wakeup(struct bdi_writeback *wb)137{138	spin_lock_irq(&wb->work_lock);139	if (test_bit(WB_registered, &wb->state))140		mod_delayed_work(bdi_wq, &wb->dwork, 0);141	spin_unlock_irq(&wb->work_lock);142}143 144/*145 * This function is used when the first inode for this wb is marked dirty. It146 * wakes-up the corresponding bdi thread which should then take care of the147 * periodic background write-out of dirty inodes. Since the write-out would148 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just149 * set up a timer which wakes the bdi thread up later.150 *151 * Note, we wouldn't bother setting up the timer, but this function is on the152 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches153 * by delaying the wake-up.154 *155 * We have to be careful not to postpone flush work if it is scheduled for156 * earlier. Thus we use queue_delayed_work().157 */158static HWJS_SUSPENDS void wb_wakeup_delayed(struct bdi_writeback *wb)159{160	unsigned long timeout;161 162	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);163	spin_lock_irq(&wb->work_lock);164	if (test_bit(WB_registered, &wb->state))165		queue_delayed_work(bdi_wq, &wb->dwork, timeout);166	spin_unlock_irq(&wb->work_lock);167}168 169static HWJS_SUSPENDS void finish_writeback_work(struct wb_writeback_work *work)170{171	struct wb_completion *done = work->done;172 173	if (work->auto_free)174		kfree(work);175	if (done) {176		wait_queue_head_t *waitq = done->waitq;177 178		/* @done can't be accessed after the following dec */179		if (atomic_dec_and_test(&done->cnt))180			wake_up_all(waitq);181	}182}183 184static HWJS_SUSPENDS void wb_queue_work(struct bdi_writeback *wb,185			  struct wb_writeback_work *work)186{187	trace_writeback_queue(wb, work);188 189	if (work->done)190		atomic_inc(&work->done->cnt);191 192	spin_lock_irq(&wb->work_lock);193 194	if (test_bit(WB_registered, &wb->state)) {195		list_add_tail(&work->list, &wb->work_list);196		mod_delayed_work(bdi_wq, &wb->dwork, 0);197	} else198		finish_writeback_work(work);199 200	spin_unlock_irq(&wb->work_lock);201}202 203/**204 * wb_wait_for_completion - wait for completion of bdi_writeback_works205 * @done: target wb_completion206 *207 * Wait for one or more work items issued to @bdi with their ->done field208 * set to @done, which should have been initialized with209 * DEFINE_WB_COMPLETION().  This function returns after all such work items210 * are completed.  Work items which are waited upon aren't freed211 * automatically on completion.212 */213void wb_wait_for_completion(struct wb_completion *done)214{215	atomic_dec(&done->cnt);		/* put down the initial count */216	wait_event(*done->waitq, !atomic_read(&done->cnt));217}218 219#ifdef CONFIG_CGROUP_WRITEBACK220 221/*222 * Parameters for foreign inode detection, see wbc_detach_inode() to see223 * how they're used.224 *225 * These paramters are inherently heuristical as the detection target226 * itself is fuzzy.  All we want to do is detaching an inode from the227 * current owner if it's being written to by some other cgroups too much.228 *229 * The current cgroup writeback is built on the assumption that multiple230 * cgroups writing to the same inode concurrently is very rare and a mode231 * of operation which isn't well supported.  As such, the goal is not232 * taking too long when a different cgroup takes over an inode while233 * avoiding too aggressive flip-flops from occasional foreign writes.234 *235 * We record, very roughly, 2s worth of IO time history and if more than236 * half of that is foreign, trigger the switch.  The recording is quantized237 * to 16 slots.  To avoid tiny writes from swinging the decision too much,238 * writes smaller than 1/8 of avg size are ignored.239 */240#define WB_FRN_TIME_SHIFT	13	/* 1s = 2^13, upto 8 secs w/ 16bit */241#define WB_FRN_TIME_AVG_SHIFT	3	/* avg = avg * 7/8 + new * 1/8 */242#define WB_FRN_TIME_CUT_DIV	8	/* ignore rounds < avg / 8 */243#define WB_FRN_TIME_PERIOD	(2 * (1 << WB_FRN_TIME_SHIFT))	/* 2s */244 245#define WB_FRN_HIST_SLOTS	16	/* inode->i_wb_frn_history is 16bit */246#define WB_FRN_HIST_UNIT	(WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS)247					/* each slot's duration is 2s / 16 */248#define WB_FRN_HIST_THR_SLOTS	(WB_FRN_HIST_SLOTS / 2)249					/* if foreign slots >= 8, switch */250#define WB_FRN_HIST_MAX_SLOTS	(WB_FRN_HIST_THR_SLOTS / 2 + 1)251					/* one round can affect upto 5 slots */252#define WB_FRN_MAX_IN_FLIGHT	1024	/* don't queue too many concurrently */253 254/*255 * Maximum inodes per isw.  A specific value has been chosen to make256 * struct inode_switch_wbs_context fit into 1024 bytes kmalloc.257 */258#define WB_MAX_INODES_PER_ISW  ((1024UL - sizeof(struct inode_switch_wbs_context)) \259                                / sizeof(struct inode *))260 261static atomic_t isw_nr_in_flight = ATOMIC_INIT(0);262static struct workqueue_struct *isw_wq;263 264void __inode_attach_wb(struct inode *inode, struct folio *folio)265{266	struct backing_dev_info *bdi = inode_to_bdi(inode);267	struct bdi_writeback *wb = NULL;268 269	if (inode_cgwb_enabled(inode)) {270		struct cgroup_subsys_state *memcg_css;271 272		if (folio) {273			memcg_css = mem_cgroup_css_from_folio(folio);274			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);275		} else {276			/* must pin memcg_css, see wb_get_create() */277			memcg_css = task_get_css(current, memory_cgrp_id);278			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);279			css_put(memcg_css);280		}281	}282 283	if (!wb)284		wb = &bdi->wb;285 286	/*287	 * There may be multiple instances of this function racing to288	 * update the same inode.  Use cmpxchg() to tell the winner.289	 */290	if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))291		wb_put(wb);292}293EXPORT_SYMBOL_GPL(__inode_attach_wb);294 295/**296 * inode_cgwb_move_to_attached - put the inode onto wb->b_attached list297 * @inode: inode of interest with i_lock held298 * @wb: target bdi_writeback299 *300 * Remove the inode from wb's io lists and if necessarily put onto b_attached301 * list.  Only inodes attached to cgwb's are kept on this list.302 */303static void inode_cgwb_move_to_attached(struct inode *inode,304					struct bdi_writeback *wb)305{306	assert_spin_locked(&wb->list_lock);307	assert_spin_locked(&inode->i_lock);308	WARN_ON_ONCE(inode->i_state & I_FREEING);309 310	inode->i_state &= ~I_SYNC_QUEUED;311	if (wb != &wb->bdi->wb)312		list_move(&inode->i_io_list, &wb->b_attached);313	else314		list_del_init(&inode->i_io_list);315	wb_io_lists_depopulated(wb);316}317 318/**319 * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it320 * @inode: inode of interest with i_lock held321 *322 * Returns @inode's wb with its list_lock held.  @inode->i_lock must be323 * held on entry and is released on return.  The returned wb is guaranteed324 * to stay @inode's associated wb until its list_lock is released.325 */326static struct bdi_writeback *327locked_inode_to_wb_and_lock_list(struct inode *inode)328	__releases(&inode->i_lock)329	__acquires(&wb->list_lock)330{331	while (true) {332		struct bdi_writeback *wb = inode_to_wb(inode);333 334		/*335		 * inode_to_wb() association is protected by both336		 * @inode->i_lock and @wb->list_lock but list_lock nests337		 * outside i_lock.  Drop i_lock and verify that the338		 * association hasn't changed after acquiring list_lock.339		 */340		wb_get(wb);341		spin_unlock(&inode->i_lock);342		spin_lock(&wb->list_lock);343 344		/* i_wb may have changed inbetween, can't use inode_to_wb() */345		if (likely(wb == inode->i_wb)) {346			wb_put(wb);	/* @inode already has ref */347			return wb;348		}349 350		spin_unlock(&wb->list_lock);351		wb_put(wb);352		cpu_relax();353		spin_lock(&inode->i_lock);354	}355}356 357/**358 * inode_to_wb_and_lock_list - determine an inode's wb and lock it359 * @inode: inode of interest360 *361 * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held362 * on entry.363 */364static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)365	__acquires(&wb->list_lock)366{367	spin_lock(&inode->i_lock);368	return locked_inode_to_wb_and_lock_list(inode);369}370 371struct inode_switch_wbs_context {372	struct rcu_work		work;373 374	/*375	 * Multiple inodes can be switched at once.  The switching procedure376	 * consists of two parts, separated by a RCU grace period.  To make377	 * sure that the second part is executed for each inode gone through378	 * the first part, all inode pointers are placed into a NULL-terminated379	 * array embedded into struct inode_switch_wbs_context.  Otherwise380	 * an inode could be left in a non-consistent state.381	 */382	struct bdi_writeback	*new_wb;383	struct inode		*inodes[];384};385 386static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi)387{388	down_write(&bdi->wb_switch_rwsem);389}390 391static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi)392{393	up_write(&bdi->wb_switch_rwsem);394}395 396static bool inode_do_switch_wbs(struct inode *inode,397				struct bdi_writeback *old_wb,398				struct bdi_writeback *new_wb)399{400	struct address_space *mapping = inode->i_mapping;401	XA_STATE(xas, &mapping->i_pages, 0);402	struct folio *folio;403	bool switched = false;404 405	spin_lock(&inode->i_lock);406	xa_lock_irq(&mapping->i_pages);407 408	/*409	 * Once I_FREEING or I_WILL_FREE are visible under i_lock, the eviction410	 * path owns the inode and we shouldn't modify ->i_io_list.411	 */412	if (unlikely(inode->i_state & (I_FREEING | I_WILL_FREE)))413		goto skip_switch;414 415	trace_inode_switch_wbs(inode, old_wb, new_wb);416 417	/*418	 * Count and transfer stats.  Note that PAGECACHE_TAG_DIRTY points419	 * to possibly dirty folios while PAGECACHE_TAG_WRITEBACK points to420	 * folios actually under writeback.421	 */422	xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_DIRTY) {423		if (folio_test_dirty(folio)) {424			long nr = folio_nr_pages(folio);425			wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr);426			wb_stat_mod(new_wb, WB_RECLAIMABLE, nr);427		}428	}429 430	xas_set(&xas, 0);431	xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) {432		long nr = folio_nr_pages(folio);433		WARN_ON_ONCE(!folio_test_writeback(folio));434		wb_stat_mod(old_wb, WB_WRITEBACK, -nr);435		wb_stat_mod(new_wb, WB_WRITEBACK, nr);436	}437 438	if (mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) {439		atomic_dec(&old_wb->writeback_inodes);440		atomic_inc(&new_wb->writeback_inodes);441	}442 443	wb_get(new_wb);444 445	/*446	 * Transfer to @new_wb's IO list if necessary.  If the @inode is dirty,447	 * the specific list @inode was on is ignored and the @inode is put on448	 * ->b_dirty which is always correct including from ->b_dirty_time.449	 * The transfer preserves @inode->dirtied_when ordering.  If the @inode450	 * was clean, it means it was on the b_attached list, so move it onto451	 * the b_attached list of @new_wb.452	 */453	if (!list_empty(&inode->i_io_list)) {454		inode->i_wb = new_wb;455 456		if (inode->i_state & I_DIRTY_ALL) {457			struct inode *pos;458 459			list_for_each_entry(pos, &new_wb->b_dirty, i_io_list)460				if (time_after_eq(inode->dirtied_when,461						  pos->dirtied_when))462					break;463			inode_io_list_move_locked(inode, new_wb,464						  pos->i_io_list.prev);465		} else {466			inode_cgwb_move_to_attached(inode, new_wb);467		}468	} else {469		inode->i_wb = new_wb;470	}471 472	/* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */473	inode->i_wb_frn_winner = 0;474	inode->i_wb_frn_avg_time = 0;475	inode->i_wb_frn_history = 0;476	switched = true;477skip_switch:478	/*479	 * Paired with load_acquire in unlocked_inode_to_wb_begin() and480	 * ensures that the new wb is visible if they see !I_WB_SWITCH.481	 */482	smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH);483 484	xa_unlock_irq(&mapping->i_pages);485	spin_unlock(&inode->i_lock);486 487	return switched;488}489 490static void inode_switch_wbs_work_fn(struct work_struct *work)491{492	struct inode_switch_wbs_context *isw =493		container_of(to_rcu_work(work), struct inode_switch_wbs_context, work);494	struct backing_dev_info *bdi = inode_to_bdi(isw->inodes[0]);495	struct bdi_writeback *old_wb = isw->inodes[0]->i_wb;496	struct bdi_writeback *new_wb = isw->new_wb;497	unsigned long nr_switched = 0;498	struct inode **inodep;499 500	/*501	 * If @inode switches cgwb membership while sync_inodes_sb() is502	 * being issued, sync_inodes_sb() might miss it.  Synchronize.503	 */504	down_read(&bdi->wb_switch_rwsem);505 506	/*507	 * By the time control reaches here, RCU grace period has passed508	 * since I_WB_SWITCH assertion and all wb stat update transactions509	 * between unlocked_inode_to_wb_begin/end() are guaranteed to be510	 * synchronizing against the i_pages lock.511	 *512	 * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock513	 * gives us exclusion against all wb related operations on @inode514	 * including IO list manipulations and stat updates.515	 */516	if (old_wb < new_wb) {517		spin_lock(&old_wb->list_lock);518		spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING);519	} else {520		spin_lock(&new_wb->list_lock);521		spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING);522	}523 524	for (inodep = isw->inodes; *inodep; inodep++) {525		WARN_ON_ONCE((*inodep)->i_wb != old_wb);526		if (inode_do_switch_wbs(*inodep, old_wb, new_wb))527			nr_switched++;528	}529 530	spin_unlock(&new_wb->list_lock);531	spin_unlock(&old_wb->list_lock);532 533	up_read(&bdi->wb_switch_rwsem);534 535	if (nr_switched) {536		wb_wakeup(new_wb);537		wb_put_many(old_wb, nr_switched);538	}539 540	for (inodep = isw->inodes; *inodep; inodep++)541		iput(*inodep);542	wb_put(new_wb);543	kfree(isw);544	atomic_dec(&isw_nr_in_flight);545}546 547static bool inode_prepare_wbs_switch(struct inode *inode,548				     struct bdi_writeback *new_wb)549{550	/*551	 * Paired with smp_mb() in cgroup_writeback_umount().552	 * isw_nr_in_flight must be increased before checking SB_ACTIVE and553	 * grabbing an inode, otherwise isw_nr_in_flight can be observed as 0554	 * in cgroup_writeback_umount() and the isw_wq will be not flushed.555	 */556	smp_mb();557 558	if (IS_DAX(inode))559		return false;560 561	/* while holding I_WB_SWITCH, no one else can update the association */562	spin_lock(&inode->i_lock);563	if (!(inode->i_sb->s_flags & SB_ACTIVE) ||564	    inode->i_state & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) ||565	    inode_to_wb(inode) == new_wb) {566		spin_unlock(&inode->i_lock);567		return false;568	}569	inode->i_state |= I_WB_SWITCH;570	__iget(inode);571	spin_unlock(&inode->i_lock);572 573	return true;574}575 576/**577 * inode_switch_wbs - change the wb association of an inode578 * @inode: target inode579 * @new_wb_id: ID of the new wb580 *581 * Switch @inode's wb association to the wb identified by @new_wb_id.  The582 * switching is performed asynchronously and may fail silently.583 */584static void inode_switch_wbs(struct inode *inode, int new_wb_id)585{586	struct backing_dev_info *bdi = inode_to_bdi(inode);587	struct cgroup_subsys_state *memcg_css;588	struct inode_switch_wbs_context *isw;589 590	/* noop if seems to be already in progress */591	if (inode->i_state & I_WB_SWITCH)592		return;593 594	/* avoid queueing a new switch if too many are already in flight */595	if (atomic_read(&isw_nr_in_flight) > WB_FRN_MAX_IN_FLIGHT)596		return;597 598	isw = kzalloc(struct_size(isw, inodes, 2), GFP_ATOMIC);599	if (!isw)600		return;601 602	atomic_inc(&isw_nr_in_flight);603 604	/* find and pin the new wb */605	rcu_read_lock();606	memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys);607	if (memcg_css && !css_tryget(memcg_css))608		memcg_css = NULL;609	rcu_read_unlock();610	if (!memcg_css)611		goto out_free;612 613	isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);614	css_put(memcg_css);615	if (!isw->new_wb)616		goto out_free;617 618	if (!inode_prepare_wbs_switch(inode, isw->new_wb))619		goto out_free;620 621	isw->inodes[0] = inode;622 623	/*624	 * In addition to synchronizing among switchers, I_WB_SWITCH tells625	 * the RCU protected stat update paths to grab the i_page626	 * lock so that stat transfer can synchronize against them.627	 * Let's continue after I_WB_SWITCH is guaranteed to be visible.628	 */629	INIT_RCU_WORK(&isw->work, inode_switch_wbs_work_fn);630	queue_rcu_work(isw_wq, &isw->work);631	return;632 633out_free:634	atomic_dec(&isw_nr_in_flight);635	if (isw->new_wb)636		wb_put(isw->new_wb);637	kfree(isw);638}639 640static bool isw_prepare_wbs_switch(struct inode_switch_wbs_context *isw,641				   struct list_head *list, int *nr)642{643	struct inode *inode;644 645	list_for_each_entry(inode, list, i_io_list) {646		if (!inode_prepare_wbs_switch(inode, isw->new_wb))647			continue;648 649		isw->inodes[*nr] = inode;650		(*nr)++;651 652		if (*nr >= WB_MAX_INODES_PER_ISW - 1)653			return true;654	}655	return false;656}657 658/**659 * cleanup_offline_cgwb - detach associated inodes660 * @wb: target wb661 *662 * Switch all inodes attached to @wb to a nearest living ancestor's wb in order663 * to eventually release the dying @wb.  Returns %true if not all inodes were664 * switched and the function has to be restarted.665 */666bool cleanup_offline_cgwb(struct bdi_writeback *wb)667{668	struct cgroup_subsys_state *memcg_css;669	struct inode_switch_wbs_context *isw;670	int nr;671	bool restart = false;672 673	isw = kzalloc(struct_size(isw, inodes, WB_MAX_INODES_PER_ISW),674		      GFP_KERNEL);675	if (!isw)676		return restart;677 678	atomic_inc(&isw_nr_in_flight);679 680	for (memcg_css = wb->memcg_css->parent; memcg_css;681	     memcg_css = memcg_css->parent) {682		isw->new_wb = wb_get_create(wb->bdi, memcg_css, GFP_KERNEL);683		if (isw->new_wb)684			break;685	}686	if (unlikely(!isw->new_wb))687		isw->new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */688 689	nr = 0;690	spin_lock(&wb->list_lock);691	/*692	 * In addition to the inodes that have completed writeback, also switch693	 * cgwbs for those inodes only with dirty timestamps. Otherwise, those694	 * inodes won't be written back for a long time when lazytime is695	 * enabled, and thus pinning the dying cgwbs. It won't break the696	 * bandwidth restrictions, as writeback of inode metadata is not697	 * accounted for.698	 */699	restart = isw_prepare_wbs_switch(isw, &wb->b_attached, &nr);700	if (!restart)701		restart = isw_prepare_wbs_switch(isw, &wb->b_dirty_time, &nr);702	spin_unlock(&wb->list_lock);703 704	/* no attached inodes? bail out */705	if (nr == 0) {706		atomic_dec(&isw_nr_in_flight);707		wb_put(isw->new_wb);708		kfree(isw);709		return restart;710	}711 712	/*713	 * In addition to synchronizing among switchers, I_WB_SWITCH tells714	 * the RCU protected stat update paths to grab the i_page715	 * lock so that stat transfer can synchronize against them.716	 * Let's continue after I_WB_SWITCH is guaranteed to be visible.717	 */718	INIT_RCU_WORK(&isw->work, inode_switch_wbs_work_fn);719	queue_rcu_work(isw_wq, &isw->work);720 721	return restart;722}723 724/**725 * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it726 * @wbc: writeback_control of interest727 * @inode: target inode728 *729 * @inode is locked and about to be written back under the control of @wbc.730 * Record @inode's writeback context into @wbc and unlock the i_lock.  On731 * writeback completion, wbc_detach_inode() should be called.  This is used732 * to track the cgroup writeback context.733 */734void wbc_attach_and_unlock_inode(struct writeback_control *wbc,735				 struct inode *inode)736{737	if (!inode_cgwb_enabled(inode)) {738		spin_unlock(&inode->i_lock);739		return;740	}741 742	wbc->wb = inode_to_wb(inode);743	wbc->inode = inode;744 745	wbc->wb_id = wbc->wb->memcg_css->id;746	wbc->wb_lcand_id = inode->i_wb_frn_winner;747	wbc->wb_tcand_id = 0;748	wbc->wb_bytes = 0;749	wbc->wb_lcand_bytes = 0;750	wbc->wb_tcand_bytes = 0;751 752	wb_get(wbc->wb);753	spin_unlock(&inode->i_lock);754 755	/*756	 * A dying wb indicates that either the blkcg associated with the757	 * memcg changed or the associated memcg is dying.  In the first758	 * case, a replacement wb should already be available and we should759	 * refresh the wb immediately.  In the second case, trying to760	 * refresh will keep failing.761	 */762	if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css)))763		inode_switch_wbs(inode, wbc->wb_id);764}765EXPORT_SYMBOL_GPL(wbc_attach_and_unlock_inode);766 767/**768 * wbc_detach_inode - disassociate wbc from inode and perform foreign detection769 * @wbc: writeback_control of the just finished writeback770 *771 * To be called after a writeback attempt of an inode finishes and undoes772 * wbc_attach_and_unlock_inode().  Can be called under any context.773 *774 * As concurrent write sharing of an inode is expected to be very rare and775 * memcg only tracks page ownership on first-use basis severely confining776 * the usefulness of such sharing, cgroup writeback tracks ownership777 * per-inode.  While the support for concurrent write sharing of an inode778 * is deemed unnecessary, an inode being written to by different cgroups at779 * different points in time is a lot more common, and, more importantly,780 * charging only by first-use can too readily lead to grossly incorrect781 * behaviors (single foreign page can lead to gigabytes of writeback to be782 * incorrectly attributed).783 *784 * To resolve this issue, cgroup writeback detects the majority dirtier of785 * an inode and transfers the ownership to it.  To avoid unnecessary786 * oscillation, the detection mechanism keeps track of history and gives787 * out the switch verdict only if the foreign usage pattern is stable over788 * a certain amount of time and/or writeback attempts.789 *790 * On each writeback attempt, @wbc tries to detect the majority writer791 * using Boyer-Moore majority vote algorithm.  In addition to the byte792 * count from the majority voting, it also counts the bytes written for the793 * current wb and the last round's winner wb (max of last round's current794 * wb, the winner from two rounds ago, and the last round's majority795 * candidate).  Keeping track of the historical winner helps the algorithm796 * to semi-reliably detect the most active writer even when it's not the797 * absolute majority.798 *799 * Once the winner of the round is determined, whether the winner is800 * foreign or not and how much IO time the round consumed is recorded in801 * inode->i_wb_frn_history.  If the amount of recorded foreign IO time is802 * over a certain threshold, the switch verdict is given.803 */804void wbc_detach_inode(struct writeback_control *wbc)805{806	struct bdi_writeback *wb = wbc->wb;807	struct inode *inode = wbc->inode;808	unsigned long avg_time, max_bytes, max_time;809	u16 history;810	int max_id;811 812	if (!wb)813		return;814 815	history = inode->i_wb_frn_history;816	avg_time = inode->i_wb_frn_avg_time;817 818	/* pick the winner of this round */819	if (wbc->wb_bytes >= wbc->wb_lcand_bytes &&820	    wbc->wb_bytes >= wbc->wb_tcand_bytes) {821		max_id = wbc->wb_id;822		max_bytes = wbc->wb_bytes;823	} else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) {824		max_id = wbc->wb_lcand_id;825		max_bytes = wbc->wb_lcand_bytes;826	} else {827		max_id = wbc->wb_tcand_id;828		max_bytes = wbc->wb_tcand_bytes;829	}830 831	/*832	 * Calculate the amount of IO time the winner consumed and fold it833	 * into the running average kept per inode.  If the consumed IO834	 * time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for835	 * deciding whether to switch or not.  This is to prevent one-off836	 * small dirtiers from skewing the verdict.837	 */838	max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT,839				wb->avg_write_bandwidth);840	if (avg_time)841		avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) -842			    (avg_time >> WB_FRN_TIME_AVG_SHIFT);843	else844		avg_time = max_time;	/* immediate catch up on first run */845 846	if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) {847		int slots;848 849		/*850		 * The switch verdict is reached if foreign wb's consume851		 * more than a certain proportion of IO time in a852		 * WB_FRN_TIME_PERIOD.  This is loosely tracked by 16 slot853		 * history mask where each bit represents one sixteenth of854		 * the period.  Determine the number of slots to shift into855		 * history from @max_time.856		 */857		slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT),858			    (unsigned long)WB_FRN_HIST_MAX_SLOTS);859		history <<= slots;860		if (wbc->wb_id != max_id)861			history |= (1U << slots) - 1;862 863		if (history)864			trace_inode_foreign_history(inode, wbc, history);865 866		/*867		 * Switch if the current wb isn't the consistent winner.868		 * If there are multiple closely competing dirtiers, the869		 * inode may switch across them repeatedly over time, which870		 * is okay.  The main goal is avoiding keeping an inode on871		 * the wrong wb for an extended period of time.872		 */873		if (hweight16(history) > WB_FRN_HIST_THR_SLOTS)874			inode_switch_wbs(inode, max_id);875	}876 877	/*878	 * Multiple instances of this function may race to update the879	 * following fields but we don't mind occassional inaccuracies.880	 */881	inode->i_wb_frn_winner = max_id;882	inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX);883	inode->i_wb_frn_history = history;884 885	wb_put(wbc->wb);886	wbc->wb = NULL;887}888EXPORT_SYMBOL_GPL(wbc_detach_inode);889 890/**891 * wbc_account_cgroup_owner - account writeback to update inode cgroup ownership892 * @wbc: writeback_control of the writeback in progress893 * @page: page being written out894 * @bytes: number of bytes being written out895 *896 * @bytes from @page are about to written out during the writeback897 * controlled by @wbc.  Keep the book for foreign inode detection.  See898 * wbc_detach_inode().899 */900void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page,901			      size_t bytes)902{903	struct folio *folio;904	struct cgroup_subsys_state *css;905	int id;906 907	/*908	 * pageout() path doesn't attach @wbc to the inode being written909	 * out.  This is intentional as we don't want the function to block910	 * behind a slow cgroup.  Ultimately, we want pageout() to kick off911	 * regular writeback instead of writing things out itself.912	 */913	if (!wbc->wb || wbc->no_cgroup_owner)914		return;915 916	folio = page_folio(page);917	css = mem_cgroup_css_from_folio(folio);918	/* dead cgroups shouldn't contribute to inode ownership arbitration */919	if (!(css->flags & CSS_ONLINE))920		return;921 922	id = css->id;923 924	if (id == wbc->wb_id) {925		wbc->wb_bytes += bytes;926		return;927	}928 929	if (id == wbc->wb_lcand_id)930		wbc->wb_lcand_bytes += bytes;931 932	/* Boyer-Moore majority vote algorithm */933	if (!wbc->wb_tcand_bytes)934		wbc->wb_tcand_id = id;935	if (id == wbc->wb_tcand_id)936		wbc->wb_tcand_bytes += bytes;937	else938		wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes);939}940EXPORT_SYMBOL_GPL(wbc_account_cgroup_owner);941 942/**943 * wb_split_bdi_pages - split nr_pages to write according to bandwidth944 * @wb: target bdi_writeback to split @nr_pages to945 * @nr_pages: number of pages to write for the whole bdi946 *947 * Split @wb's portion of @nr_pages according to @wb's write bandwidth in948 * relation to the total write bandwidth of all wb's w/ dirty inodes on949 * @wb->bdi.950 */951static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)952{953	unsigned long this_bw = wb->avg_write_bandwidth;954	unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);955 956	if (nr_pages == LONG_MAX)957		return LONG_MAX;958 959	/*960	 * This may be called on clean wb's and proportional distribution961	 * may not make sense, just use the original @nr_pages in those962	 * cases.  In general, we wanna err on the side of writing more.963	 */964	if (!tot_bw || this_bw >= tot_bw)965		return nr_pages;966	else967		return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw);968}969 970/**971 * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi972 * @bdi: target backing_dev_info973 * @base_work: wb_writeback_work to issue974 * @skip_if_busy: skip wb's which already have writeback in progress975 *976 * Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which977 * have dirty inodes.  If @base_work->nr_page isn't %LONG_MAX, it's978 * distributed to the busy wbs according to each wb's proportion in the979 * total active write bandwidth of @bdi.980 */981static HWJS_SUSPENDS void bdi_split_work_to_wbs(struct backing_dev_info *bdi,982				  struct wb_writeback_work *base_work,983				  bool skip_if_busy)984{985	struct bdi_writeback *last_wb = NULL;986	struct bdi_writeback *wb = list_entry(&bdi->wb_list,987					      struct bdi_writeback, bdi_node);988 989	might_sleep();990restart:991	rcu_read_lock();992	list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) {993		DEFINE_WB_COMPLETION(fallback_work_done, bdi);994		struct wb_writeback_work fallback_work;995		struct wb_writeback_work *work;996		long nr_pages;997 998		if (last_wb) {999			wb_put(last_wb);1000			last_wb = NULL;1001		}1002 1003		/* SYNC_ALL writes out I_DIRTY_TIME too */1004		if (!wb_has_dirty_io(wb) &&1005		    (base_work->sync_mode == WB_SYNC_NONE ||1006		     list_empty(&wb->b_dirty_time)))1007			continue;1008		if (skip_if_busy && writeback_in_progress(wb))1009			continue;1010 1011		nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);1012 1013		work = kmalloc(sizeof(*work), GFP_ATOMIC);1014		if (work) {1015			*work = *base_work;1016			work->nr_pages = nr_pages;1017			work->auto_free = 1;1018			wb_queue_work(wb, work);1019			continue;1020		}1021 1022		/*1023		 * If wb_tryget fails, the wb has been shutdown, skip it.1024		 *1025		 * Pin @wb so that it stays on @bdi->wb_list.  This allows1026		 * continuing iteration from @wb after dropping and1027		 * regrabbing rcu read lock.1028		 */1029		if (!wb_tryget(wb))1030			continue;1031 1032		/* alloc failed, execute synchronously using on-stack fallback */1033		work = &fallback_work;1034		*work = *base_work;1035		work->nr_pages = nr_pages;1036		work->auto_free = 0;1037		work->done = &fallback_work_done;1038 1039		wb_queue_work(wb, work);1040		last_wb = wb;1041 1042		rcu_read_unlock();1043		wb_wait_for_completion(&fallback_work_done);1044		goto restart;1045	}1046	rcu_read_unlock();1047 1048	if (last_wb)1049		wb_put(last_wb);1050}1051 1052/**1053 * cgroup_writeback_by_id - initiate cgroup writeback from bdi and memcg IDs1054 * @bdi_id: target bdi id1055 * @memcg_id: target memcg css id1056 * @reason: reason why some writeback work initiated1057 * @done: target wb_completion1058 *1059 * Initiate flush of the bdi_writeback identified by @bdi_id and @memcg_id1060 * with the specified parameters.1061 */1062int cgroup_writeback_by_id(u64 bdi_id, int memcg_id,1063			   enum wb_reason reason, struct wb_completion *done)1064{1065	struct backing_dev_info *bdi;1066	struct cgroup_subsys_state *memcg_css;1067	struct bdi_writeback *wb;1068	struct wb_writeback_work *work;1069	unsigned long dirty;1070	int ret;1071 1072	/* lookup bdi and memcg */1073	bdi = bdi_get_by_id(bdi_id);1074	if (!bdi)1075		return -ENOENT;1076 1077	rcu_read_lock();1078	memcg_css = css_from_id(memcg_id, &memory_cgrp_subsys);1079	if (memcg_css && !css_tryget(memcg_css))1080		memcg_css = NULL;1081	rcu_read_unlock();1082	if (!memcg_css) {1083		ret = -ENOENT;1084		goto out_bdi_put;1085	}1086 1087	/*1088	 * And find the associated wb.  If the wb isn't there already1089	 * there's nothing to flush, don't create one.1090	 */1091	wb = wb_get_lookup(bdi, memcg_css);1092	if (!wb) {1093		ret = -ENOENT;1094		goto out_css_put;1095	}1096 1097	/*1098	 * The caller is attempting to write out most of1099	 * the currently dirty pages.  Let's take the current dirty page1100	 * count and inflate it by 25% which should be large enough to1101	 * flush out most dirty pages while avoiding getting livelocked by1102	 * concurrent dirtiers.1103	 *1104	 * BTW the memcg stats are flushed periodically and this is best-effort1105	 * estimation, so some potential error is ok.1106	 */1107	dirty = memcg_page_state(mem_cgroup_from_css(memcg_css), NR_FILE_DIRTY);1108	dirty = dirty * 10 / 8;1109 1110	/* issue the writeback work */1111	work = kzalloc(sizeof(*work), GFP_NOWAIT | __GFP_NOWARN);1112	if (work) {1113		work->nr_pages = dirty;1114		work->sync_mode = WB_SYNC_NONE;1115		work->range_cyclic = 1;1116		work->reason = reason;1117		work->done = done;1118		work->auto_free = 1;1119		wb_queue_work(wb, work);1120		ret = 0;1121	} else {1122		ret = -ENOMEM;1123	}1124 1125	wb_put(wb);1126out_css_put:1127	css_put(memcg_css);1128out_bdi_put:1129	bdi_put(bdi);1130	return ret;1131}1132 1133/**1134 * cgroup_writeback_umount - flush inode wb switches for umount1135 * @sb: target super_block1136 *1137 * This function is called when a super_block is about to be destroyed and1138 * flushes in-flight inode wb switches.  An inode wb switch goes through1139 * RCU and then workqueue, so the two need to be flushed in order to ensure1140 * that all previously scheduled switches are finished.  As wb switches are1141 * rare occurrences and synchronize_rcu() can take a while, perform1142 * flushing iff wb switches are in flight.1143 */1144void cgroup_writeback_umount(struct super_block *sb)1145{1146 1147	if (!(sb->s_bdi->capabilities & BDI_CAP_WRITEBACK))1148		return;1149 1150	/*1151	 * SB_ACTIVE should be reliably cleared before checking1152	 * isw_nr_in_flight, see generic_shutdown_super().1153	 */1154	smp_mb();1155 1156	if (atomic_read(&isw_nr_in_flight)) {1157		/*1158		 * Use rcu_barrier() to wait for all pending callbacks to1159		 * ensure that all in-flight wb switches are in the workqueue.1160		 */1161		rcu_barrier();1162		flush_workqueue(isw_wq);1163	}1164}1165 1166static int __init cgroup_writeback_init(void)1167{1168	isw_wq = alloc_workqueue("inode_switch_wbs", 0, 0);1169	if (!isw_wq)1170		return -ENOMEM;1171	return 0;1172}1173fs_initcall(cgroup_writeback_init);1174 1175#else	/* CONFIG_CGROUP_WRITEBACK */1176 1177static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }1178static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }1179 1180static void inode_cgwb_move_to_attached(struct inode *inode,1181					struct bdi_writeback *wb)1182{1183	assert_spin_locked(&wb->list_lock);1184	assert_spin_locked(&inode->i_lock);1185	WARN_ON_ONCE(inode->i_state & I_FREEING);1186 1187	inode->i_state &= ~I_SYNC_QUEUED;1188	list_del_init(&inode->i_io_list);1189	wb_io_lists_depopulated(wb);1190}1191 1192static struct bdi_writeback *1193locked_inode_to_wb_and_lock_list(struct inode *inode)1194	__releases(&inode->i_lock)1195	__acquires(&wb->list_lock)1196{1197	struct bdi_writeback *wb = inode_to_wb(inode);1198 1199	spin_unlock(&inode->i_lock);1200	spin_lock(&wb->list_lock);1201	return wb;1202}1203 1204static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)1205	__acquires(&wb->list_lock)1206{1207	struct bdi_writeback *wb = inode_to_wb(inode);1208 1209	spin_lock(&wb->list_lock);1210	return wb;1211}1212 1213static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)1214{1215	return nr_pages;1216}1217 1218static HWJS_SUSPENDS void bdi_split_work_to_wbs(struct backing_dev_info *bdi,1219				  struct wb_writeback_work *base_work,1220				  bool skip_if_busy)1221{1222	might_sleep();1223 1224	if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {1225		base_work->auto_free = 0;1226		wb_queue_work(&bdi->wb, base_work);1227	}1228}1229 1230#endif	/* CONFIG_CGROUP_WRITEBACK */1231 1232/*1233 * Add in the number of potentially dirty inodes, because each inode1234 * write can dirty pagecache in the underlying blockdev.1235 */1236static unsigned long get_nr_dirty_pages(void)1237{1238	return global_node_page_state(NR_FILE_DIRTY) +1239		get_nr_dirty_inodes();1240}1241 1242static HWJS_SUSPENDS void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason)1243{1244	if (!wb_has_dirty_io(wb))1245		return;1246 1247	/*1248	 * All callers of this function want to start writeback of all1249	 * dirty pages. Places like vmscan can call this at a very1250	 * high frequency, causing pointless allocations of tons of1251	 * work items and keeping the flusher threads busy retrieving1252	 * that work. Ensure that we only allow one of them pending and1253	 * inflight at the time.1254	 */1255	if (test_bit(WB_start_all, &wb->state) ||1256	    test_and_set_bit(WB_start_all, &wb->state))1257		return;1258 1259	wb->start_all_reason = reason;1260	wb_wakeup(wb);1261}1262 1263/**1264 * wb_start_background_writeback - start background writeback1265 * @wb: bdi_writback to write from1266 *1267 * Description:1268 *   This makes sure WB_SYNC_NONE background writeback happens. When1269 *   this function returns, it is only guaranteed that for given wb1270 *   some IO is happening if we are over background dirty threshold.1271 *   Caller need not hold sb s_umount semaphore.1272 */1273void wb_start_background_writeback(struct bdi_writeback *wb)1274{1275	/*1276	 * We just wake up the flusher thread. It will perform background1277	 * writeback as soon as there is no other work to do.1278	 */1279	trace_writeback_wake_background(wb);1280	wb_wakeup(wb);1281}1282 1283/*1284 * Remove the inode from the writeback list it is on.1285 */1286void inode_io_list_del(struct inode *inode)1287{1288	struct bdi_writeback *wb;1289 1290	wb = inode_to_wb_and_lock_list(inode);1291	spin_lock(&inode->i_lock);1292 1293	inode->i_state &= ~I_SYNC_QUEUED;1294	list_del_init(&inode->i_io_list);1295	wb_io_lists_depopulated(wb);1296 1297	spin_unlock(&inode->i_lock);1298	spin_unlock(&wb->list_lock);1299}1300EXPORT_SYMBOL(inode_io_list_del);1301 1302/*1303 * mark an inode as under writeback on the sb1304 */1305void sb_mark_inode_writeback(struct inode *inode)1306{1307	struct super_block *sb = inode->i_sb;1308	unsigned long flags;1309 1310	if (list_empty(&inode->i_wb_list)) {1311		spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);1312		if (list_empty(&inode->i_wb_list)) {1313			list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb);1314			trace_sb_mark_inode_writeback(inode);1315		}1316		spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);1317	}1318}1319 1320/*1321 * clear an inode as under writeback on the sb1322 */1323void sb_clear_inode_writeback(struct inode *inode)1324{1325	struct super_block *sb = inode->i_sb;1326	unsigned long flags;1327 1328	if (!list_empty(&inode->i_wb_list)) {1329		spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);1330		if (!list_empty(&inode->i_wb_list)) {1331			list_del_init(&inode->i_wb_list);1332			trace_sb_clear_inode_writeback(inode);1333		}1334		spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);1335	}1336}1337 1338/*1339 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the1340 * furthest end of its superblock's dirty-inode list.1341 *1342 * Before stamping the inode's ->dirtied_when, we check to see whether it is1343 * already the most-recently-dirtied inode on the b_dirty list.  If that is1344 * the case then the inode must have been redirtied while it was being written1345 * out and we don't reset its dirtied_when.1346 */1347static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb)1348{1349	assert_spin_locked(&inode->i_lock);1350 1351	inode->i_state &= ~I_SYNC_QUEUED;1352	/*1353	 * When the inode is being freed just don't bother with dirty list1354	 * tracking. Flush worker will ignore this inode anyway and it will1355	 * trigger assertions in inode_io_list_move_locked().1356	 */1357	if (inode->i_state & I_FREEING) {1358		list_del_init(&inode->i_io_list);1359		wb_io_lists_depopulated(wb);1360		return;1361	}1362	if (!list_empty(&wb->b_dirty)) {1363		struct inode *tail;1364 1365		tail = wb_inode(wb->b_dirty.next);1366		if (time_before(inode->dirtied_when, tail->dirtied_when))1367			inode->dirtied_when = jiffies;1368	}1369	inode_io_list_move_locked(inode, wb, &wb->b_dirty);1370}1371 1372static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)1373{1374	spin_lock(&inode->i_lock);1375	redirty_tail_locked(inode, wb);1376	spin_unlock(&inode->i_lock);1377}1378 1379/*1380 * requeue inode for re-scanning after bdi->b_io list is exhausted.1381 */1382static void requeue_io(struct inode *inode, struct bdi_writeback *wb)1383{1384	inode_io_list_move_locked(inode, wb, &wb->b_more_io);1385}1386 1387static HWJS_SUSPENDS void inode_sync_complete(struct inode *inode)1388{1389	assert_spin_locked(&inode->i_lock);1390 1391	inode->i_state &= ~I_SYNC;1392	/* If inode is clean an unused, put it into LRU now... */1393	inode_add_lru(inode);1394	/* Called with inode->i_lock which ensures memory ordering. */1395	inode_wake_up_bit(inode, __I_SYNC);1396}1397 1398static bool inode_dirtied_after(struct inode *inode, unsigned long t)1399{1400	bool ret = time_after(inode->dirtied_when, t);1401#ifndef CONFIG_64BIT1402	/*1403	 * For inodes being constantly redirtied, dirtied_when can get stuck.1404	 * It _appears_ to be in the future, but is actually in distant past.1405	 * This test is necessary to prevent such wrapped-around relative times1406	 * from permanently stopping the whole bdi writeback.1407	 */1408	ret = ret && time_before_eq(inode->dirtied_when, jiffies);1409#endif1410	return ret;1411}1412 1413/*1414 * Move expired (dirtied before dirtied_before) dirty inodes from1415 * @delaying_queue to @dispatch_queue.1416 */1417static int move_expired_inodes(struct list_head *delaying_queue,1418			       struct list_head *dispatch_queue,1419			       unsigned long dirtied_before)1420{1421	LIST_HEAD(tmp);1422	struct list_head *pos, *node;1423	struct super_block *sb = NULL;1424	struct inode *inode;1425	int do_sb_sort = 0;1426	int moved = 0;1427 1428	while (!list_empty(delaying_queue)) {1429		inode = wb_inode(delaying_queue->prev);1430		if (inode_dirtied_after(inode, dirtied_before))1431			break;1432		spin_lock(&inode->i_lock);1433		list_move(&inode->i_io_list, &tmp);1434		moved++;1435		inode->i_state |= I_SYNC_QUEUED;1436		spin_unlock(&inode->i_lock);1437		if (sb_is_blkdev_sb(inode->i_sb))1438			continue;1439		if (sb && sb != inode->i_sb)1440			do_sb_sort = 1;1441		sb = inode->i_sb;1442	}1443 1444	/* just one sb in list, splice to dispatch_queue and we're done */1445	if (!do_sb_sort) {1446		list_splice(&tmp, dispatch_queue);1447		goto out;1448	}1449 1450	/*1451	 * Although inode's i_io_list is moved from 'tmp' to 'dispatch_queue',1452	 * we don't take inode->i_lock here because it is just a pointless overhead.1453	 * Inode is already marked as I_SYNC_QUEUED so writeback list handling is1454	 * fully under our control.1455	 */1456	while (!list_empty(&tmp)) {1457		sb = wb_inode(tmp.prev)->i_sb;1458		list_for_each_prev_safe(pos, node, &tmp) {1459			inode = wb_inode(pos);1460			if (inode->i_sb == sb)1461				list_move(&inode->i_io_list, dispatch_queue);1462		}1463	}1464out:1465	return moved;1466}1467 1468/*1469 * Queue all expired dirty inodes for io, eldest first.1470 * Before1471 *         newly dirtied     b_dirty    b_io    b_more_io1472 *         =============>    gf         edc     BA1473 * After1474 *         newly dirtied     b_dirty    b_io    b_more_io1475 *         =============>    g          fBAedc1476 *                                           |1477 *                                           +--> dequeue for IO1478 */1479static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work,1480		     unsigned long dirtied_before)1481{1482	int moved;1483	unsigned long time_expire_jif = dirtied_before;1484 1485	assert_spin_locked(&wb->list_lock);1486	list_splice_init(&wb->b_more_io, &wb->b_io);1487	moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before);1488	if (!work->for_sync)1489		time_expire_jif = jiffies - dirtytime_expire_interval * HZ;1490	moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,1491				     time_expire_jif);1492	if (moved)1493		wb_io_lists_populated(wb);1494	trace_writeback_queue_io(wb, work, dirtied_before, moved);1495}1496 1497static HWJS_SUSPENDS int write_inode(struct inode *inode, struct writeback_control *wbc)1498{1499	int ret;1500 1501	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {1502		trace_writeback_write_inode_start(inode, wbc);1503		ret = inode->i_sb->s_op->write_inode(inode, wbc);1504		trace_writeback_write_inode(inode, wbc);1505		return ret;1506	}1507	return 0;1508}1509 1510/*1511 * Wait for writeback on an inode to complete. Called with i_lock held.1512 * Caller must make sure inode cannot go away when we drop i_lock.1513 */1514void inode_wait_for_writeback(struct inode *inode)1515{1516	struct wait_bit_queue_entry wqe;1517	struct wait_queue_head *wq_head;1518 1519	assert_spin_locked(&inode->i_lock);1520 1521	if (!(inode->i_state & I_SYNC))1522		return;1523 1524	wq_head = inode_bit_waitqueue(&wqe, inode, __I_SYNC);1525	for (;;) {1526		prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE);1527		/* Checking I_SYNC with inode->i_lock guarantees memory ordering. */1528		if (!(inode->i_state & I_SYNC))1529			break;1530		spin_unlock(&inode->i_lock);1531		schedule();1532		spin_lock(&inode->i_lock);1533	}1534	finish_wait(wq_head, &wqe.wq_entry);1535}1536 1537/*1538 * Sleep until I_SYNC is cleared. This function must be called with i_lock1539 * held and drops it. It is aimed for callers not holding any inode reference1540 * so once i_lock is dropped, inode can go away.1541 */1542static HWJS_SUSPENDS void inode_sleep_on_writeback(struct inode *inode)1543	__releases(inode->i_lock)1544{1545	struct wait_bit_queue_entry wqe;1546	struct wait_queue_head *wq_head;1547	bool sleep;1548 1549	assert_spin_locked(&inode->i_lock);1550 1551	wq_head = inode_bit_waitqueue(&wqe, inode, __I_SYNC);1552	prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE);1553	/* Checking I_SYNC with inode->i_lock guarantees memory ordering. */1554	sleep = !!(inode->i_state & I_SYNC);1555	spin_unlock(&inode->i_lock);1556	if (sleep)1557		schedule();1558	finish_wait(wq_head, &wqe.wq_entry);1559}1560 1561/*1562 * Find proper writeback list for the inode depending on its current state and1563 * possibly also change of its state while we were doing writeback.  Here we1564 * handle things such as livelock prevention or fairness of writeback among1565 * inodes. This function can be called only by flusher thread - noone else1566 * processes all inodes in writeback lists and requeueing inodes behind flusher1567 * thread's back can have unexpected consequences.1568 */1569static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,1570			  struct writeback_control *wbc,1571			  unsigned long dirtied_before)1572{1573	if (inode->i_state & I_FREEING)1574		return;1575 1576	/*1577	 * Sync livelock prevention. Each inode is tagged and synced in one1578	 * shot. If still dirty, it will be redirty_tail()'ed below.  Update1579	 * the dirty time to prevent enqueue and sync it again.1580	 */1581	if ((inode->i_state & I_DIRTY) &&1582	    (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))1583		inode->dirtied_when = jiffies;1584 1585	if (wbc->pages_skipped) {1586		/*1587		 * Writeback is not making progress due to locked buffers.1588		 * Skip this inode for now. Although having skipped pages1589		 * is odd for clean inodes, it can happen for some1590		 * filesystems so handle that gracefully.1591		 */1592		if (inode->i_state & I_DIRTY_ALL)1593			redirty_tail_locked(inode, wb);1594		else1595			inode_cgwb_move_to_attached(inode, wb);1596		return;1597	}1598 1599	if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {1600		/*1601		 * We didn't write back all the pages.  nfs_writepages()1602		 * sometimes bales out without doing anything.1603		 */1604		if (wbc->nr_to_write <= 0 &&1605		    !inode_dirtied_after(inode, dirtied_before)) {1606			/* Slice used up. Queue for next turn. */1607			requeue_io(inode, wb);1608		} else {1609			/*1610			 * Writeback blocked by something other than1611			 * congestion. Delay the inode for some time to1612			 * avoid spinning on the CPU (100% iowait)1613			 * retrying writeback of the dirty page/inode1614			 * that cannot be performed immediately.1615			 */1616			redirty_tail_locked(inode, wb);1617		}1618	} else if (inode->i_state & I_DIRTY) {1619		/*1620		 * Filesystems can dirty the inode during writeback operations,1621		 * such as delayed allocation during submission or metadata1622		 * updates after data IO completion.1623		 */1624		redirty_tail_locked(inode, wb);1625	} else if (inode->i_state & I_DIRTY_TIME) {1626		inode->dirtied_when = jiffies;1627		inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);1628		inode->i_state &= ~I_SYNC_QUEUED;1629	} else {1630		/* The inode is clean. Remove from writeback lists. */1631		inode_cgwb_move_to_attached(inode, wb);1632	}1633}1634 1635/*1636 * Write out an inode and its dirty pages (or some of its dirty pages, depending1637 * on @wbc->nr_to_write), and clear the relevant dirty flags from i_state.1638 *1639 * This doesn't remove the inode from the writeback list it is on, except1640 * potentially to move it from b_dirty_time to b_dirty due to timestamp1641 * expiration.  The caller is otherwise responsible for writeback list handling.1642 *1643 * The caller is also responsible for setting the I_SYNC flag beforehand and1644 * calling inode_sync_complete() to clear it afterwards.1645 */1646static HWJS_SUSPENDS int1647__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)1648{1649	struct address_space *mapping = inode->i_mapping;1650	long nr_to_write = wbc->nr_to_write;1651	unsigned dirty;1652	int ret;1653 1654	WARN_ON(!(inode->i_state & I_SYNC));1655 1656	trace_writeback_single_inode_start(inode, wbc, nr_to_write);1657 1658	ret = do_writepages(mapping, wbc);1659 1660	/*1661	 * Make sure to wait on the data before writing out the metadata.1662	 * This is important for filesystems that modify metadata on data1663	 * I/O completion. We don't do it for sync(2) writeback because it has a1664	 * separate, external IO completion path and ->sync_fs for guaranteeing1665	 * inode metadata is written back correctly.1666	 */1667	if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {1668		int err = filemap_fdatawait(mapping);1669		if (ret == 0)1670			ret = err;1671	}1672 1673	/*1674	 * If the inode has dirty timestamps and we need to write them, call1675	 * mark_inode_dirty_sync() to notify the filesystem about it and to1676	 * change I_DIRTY_TIME into I_DIRTY_SYNC.1677	 */1678	if ((inode->i_state & I_DIRTY_TIME) &&1679	    (wbc->sync_mode == WB_SYNC_ALL ||1680	     time_after(jiffies, inode->dirtied_time_when +1681			dirtytime_expire_interval * HZ))) {1682		trace_writeback_lazytime(inode);1683		mark_inode_dirty_sync(inode);1684	}1685 1686	/*1687	 * Get and clear the dirty flags from i_state.  This needs to be done1688	 * after calling writepages because some filesystems may redirty the1689	 * inode during writepages due to delalloc.  It also needs to be done1690	 * after handling timestamp expiration, as that may dirty the inode too.1691	 */1692	spin_lock(&inode->i_lock);1693	dirty = inode->i_state & I_DIRTY;1694	inode->i_state &= ~dirty;1695 1696	/*1697	 * Paired with smp_mb() in __mark_inode_dirty().  This allows1698	 * __mark_inode_dirty() to test i_state without grabbing i_lock -1699	 * either they see the I_DIRTY bits cleared or we see the dirtied1700	 * inode.1701	 *1702	 * I_DIRTY_PAGES is always cleared together above even if @mapping1703	 * still has dirty pages.  The flag is reinstated after smp_mb() if1704	 * necessary.  This guarantees that either __mark_inode_dirty()1705	 * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.1706	 */1707	smp_mb();1708 1709	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))1710		inode->i_state |= I_DIRTY_PAGES;1711	else if (unlikely(inode->i_state & I_PINNING_NETFS_WB)) {1712		if (!(inode->i_state & I_DIRTY_PAGES)) {1713			inode->i_state &= ~I_PINNING_NETFS_WB;1714			wbc->unpinned_netfs_wb = true;1715			dirty |= I_PINNING_NETFS_WB; /* Cause write_inode */1716		}1717	}1718 1719	spin_unlock(&inode->i_lock);1720 1721	/* Don't write the inode if only I_DIRTY_PAGES was set */1722	if (dirty & ~I_DIRTY_PAGES) {1723		int err = write_inode(inode, wbc);1724		if (ret == 0)1725			ret = err;1726	}1727	wbc->unpinned_netfs_wb = false;1728	trace_writeback_single_inode(inode, wbc, nr_to_write);1729	return ret;1730}1731 1732/*1733 * Write out an inode's dirty data and metadata on-demand, i.e. separately from1734 * the regular batched writeback done by the flusher threads in1735 * writeback_sb_inodes().  @wbc controls various aspects of the write, such as1736 * whether it is a data-integrity sync (%WB_SYNC_ALL) or not (%WB_SYNC_NONE).1737 *1738 * To prevent the inode from going away, either the caller must have a reference1739 * to the inode, or the inode must have I_WILL_FREE or I_FREEING set.1740 */1741static HWJS_SUSPENDS int writeback_single_inode(struct inode *inode,1742				  struct writeback_control *wbc)1743{1744	struct bdi_writeback *wb;1745	int ret = 0;1746 1747	spin_lock(&inode->i_lock);1748	if (!atomic_read(&inode->i_count))1749		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));1750	else1751		WARN_ON(inode->i_state & I_WILL_FREE);1752 1753	if (inode->i_state & I_SYNC) {1754		/*1755		 * Writeback is already running on the inode.  For WB_SYNC_NONE,1756		 * that's enough and we can just return.  For WB_SYNC_ALL, we1757		 * must wait for the existing writeback to complete, then do1758		 * writeback again if there's anything left.1759		 */1760		if (wbc->sync_mode != WB_SYNC_ALL)1761			goto out;1762		inode_wait_for_writeback(inode);1763	}1764	WARN_ON(inode->i_state & I_SYNC);1765	/*1766	 * If the inode is already fully clean, then there's nothing to do.1767	 *1768	 * For data-integrity syncs we also need to check whether any pages are1769	 * still under writeback, e.g. due to prior WB_SYNC_NONE writeback.  If1770	 * there are any such pages, we'll need to wait for them.1771	 */1772	if (!(inode->i_state & I_DIRTY_ALL) &&1773	    (wbc->sync_mode != WB_SYNC_ALL ||1774	     !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))1775		goto out;1776	inode->i_state |= I_SYNC;1777	wbc_attach_and_unlock_inode(wbc, inode);1778 1779	ret = __writeback_single_inode(inode, wbc);1780 1781	wbc_detach_inode(wbc);1782 1783	wb = inode_to_wb_and_lock_list(inode);1784	spin_lock(&inode->i_lock);1785	/*1786	 * If the inode is freeing, its i_io_list shoudn't be updated1787	 * as it can be finally deleted at this moment.1788	 */1789	if (!(inode->i_state & I_FREEING)) {1790		/*1791		 * If the inode is now fully clean, then it can be safely1792		 * removed from its writeback list (if any). Otherwise the1793		 * flusher threads are responsible for the writeback lists.1794		 */1795		if (!(inode->i_state & I_DIRTY_ALL))1796			inode_cgwb_move_to_attached(inode, wb);1797		else if (!(inode->i_state & I_SYNC_QUEUED)) {1798			if ((inode->i_state & I_DIRTY))1799				redirty_tail_locked(inode, wb);1800			else if (inode->i_state & I_DIRTY_TIME) {1801				inode->dirtied_when = jiffies;1802				inode_io_list_move_locked(inode,1803							  wb,1804							  &wb->b_dirty_time);1805			}1806		}1807	}1808 1809	spin_unlock(&wb->list_lock);1810	inode_sync_complete(inode);1811out:1812	spin_unlock(&inode->i_lock);1813	return ret;1814}1815 1816static long writeback_chunk_size(struct bdi_writeback *wb,1817				 struct wb_writeback_work *work)1818{1819	long pages;1820 1821	/*1822	 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty1823	 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX1824	 * here avoids calling into writeback_inodes_wb() more than once.1825	 *1826	 * The intended call sequence for WB_SYNC_ALL writeback is:1827	 *1828	 *      wb_writeback()1829	 *          writeback_sb_inodes()       <== called only once1830	 *              write_cache_pages()     <== called once for each inode1831	 *                   (quickly) tag currently dirty pages1832	 *                   (maybe slowly) sync all tagged pages1833	 */1834	if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)1835		pages = LONG_MAX;1836	else {1837		pages = min(wb->avg_write_bandwidth / 2,1838			    global_wb_domain.dirty_limit / DIRTY_SCOPE);1839		pages = min(pages, work->nr_pages);1840		pages = round_down(pages + MIN_WRITEBACK_PAGES,1841				   MIN_WRITEBACK_PAGES);1842	}1843 1844	return pages;1845}1846 1847/*1848 * Write a portion of b_io inodes which belong to @sb.1849 *1850 * Return the number of pages and/or inodes written.1851 *1852 * NOTE! This is called with wb->list_lock held, and will1853 * unlock and relock that for each inode it ends up doing1854 * IO for.1855 */1856static HWJS_SUSPENDS long writeback_sb_inodes(struct super_block *sb,1857				struct bdi_writeback *wb,1858				struct wb_writeback_work *work)1859{1860	struct writeback_control wbc = {1861		.sync_mode		= work->sync_mode,1862		.tagged_writepages	= work->tagged_writepages,1863		.for_kupdate		= work->for_kupdate,1864		.for_background		= work->for_background,1865		.for_sync		= work->for_sync,1866		.range_cyclic		= work->range_cyclic,1867		.range_start		= 0,1868		.range_end		= LLONG_MAX,1869	};1870	unsigned long start_time = jiffies;1871	long write_chunk;1872	long total_wrote = 0;  /* count both pages and inodes */1873	unsigned long dirtied_before = jiffies;1874 1875	if (work->for_kupdate)1876		dirtied_before = jiffies -1877			msecs_to_jiffies(dirty_expire_interval * 10);1878 1879	while (!list_empty(&wb->b_io)) {1880		struct inode *inode = wb_inode(wb->b_io.prev);1881		struct bdi_writeback *tmp_wb;1882		long wrote;1883 1884		if (inode->i_sb != sb) {1885			if (work->sb) {1886				/*1887				 * We only want to write back data for this1888				 * superblock, move all inodes not belonging1889				 * to it back onto the dirty list.1890				 */1891				redirty_tail(inode, wb);1892				continue;1893			}1894 1895			/*1896			 * The inode belongs to a different superblock.1897			 * Bounce back to the caller to unpin this and1898			 * pin the next superblock.1899			 */1900			break;1901		}1902 1903		/*1904		 * Don't bother with new inodes or inodes being freed, first1905		 * kind does not need periodic writeout yet, and for the latter1906		 * kind writeout is handled by the freer.1907		 */1908		spin_lock(&inode->i_lock);1909		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {1910			redirty_tail_locked(inode, wb);1911			spin_unlock(&inode->i_lock);1912			continue;1913		}1914		if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {1915			/*1916			 * If this inode is locked for writeback and we are not1917			 * doing writeback-for-data-integrity, move it to1918			 * b_more_io so that writeback can proceed with the1919			 * other inodes on s_io.1920			 *1921			 * We'll have another go at writing back this inode1922			 * when we completed a full scan of b_io.1923			 */1924			requeue_io(inode, wb);1925			spin_unlock(&inode->i_lock);1926			trace_writeback_sb_inodes_requeue(inode);1927			continue;1928		}1929		spin_unlock(&wb->list_lock);1930 1931		/*1932		 * We already requeued the inode if it had I_SYNC set and we1933		 * are doing WB_SYNC_NONE writeback. So this catches only the1934		 * WB_SYNC_ALL case.1935		 */1936		if (inode->i_state & I_SYNC) {1937			/* Wait for I_SYNC. This function drops i_lock... */1938			inode_sleep_on_writeback(inode);1939			/* Inode may be gone, start again */1940			spin_lock(&wb->list_lock);1941			continue;1942		}1943		inode->i_state |= I_SYNC;1944		wbc_attach_and_unlock_inode(&wbc, inode);1945 1946		write_chunk = writeback_chunk_size(wb, work);1947		wbc.nr_to_write = write_chunk;1948		wbc.pages_skipped = 0;1949 1950		/*1951		 * We use I_SYNC to pin the inode in memory. While it is set1952		 * evict_inode() will wait so the inode cannot be freed.1953		 */1954		__writeback_single_inode(inode, &wbc);1955 1956		wbc_detach_inode(&wbc);1957		work->nr_pages -= write_chunk - wbc.nr_to_write;1958		wrote = write_chunk - wbc.nr_to_write - wbc.pages_skipped;1959		wrote = wrote < 0 ? 0 : wrote;1960		total_wrote += wrote;1961 1962		if (need_resched()) {1963			/*1964			 * We're trying to balance between building up a nice1965			 * long list of IOs to improve our merge rate, and1966			 * getting those IOs out quickly for anyone throttling1967			 * in balance_dirty_pages().  cond_resched() doesn't1968			 * unplug, so get our IOs out the door before we1969			 * give up the CPU.1970			 */1971			blk_flush_plug(current->plug, false);1972			cond_resched();1973		}1974 1975		/*1976		 * Requeue @inode if still dirty.  Be careful as @inode may1977		 * have been switched to another wb in the meantime.1978		 */1979		tmp_wb = inode_to_wb_and_lock_list(inode);1980		spin_lock(&inode->i_lock);1981		if (!(inode->i_state & I_DIRTY_ALL))1982			total_wrote++;1983		requeue_inode(inode, tmp_wb, &wbc, dirtied_before);1984		inode_sync_complete(inode);1985		spin_unlock(&inode->i_lock);1986 1987		if (unlikely(tmp_wb != wb)) {1988			spin_unlock(&tmp_wb->list_lock);1989			spin_lock(&wb->list_lock);1990		}1991 1992		/*1993		 * bail out to wb_writeback() often enough to check1994		 * background threshold and other termination conditions.1995		 */1996		if (total_wrote) {1997			if (time_is_before_jiffies(start_time + HZ / 10UL))1998				break;1999			if (work->nr_pages <= 0)2000				break;2001		}2002	}2003	return total_wrote;2004}2005 2006static HWJS_SUSPENDS long __writeback_inodes_wb(struct bdi_writeback *wb,2007				  struct wb_writeback_work *work)2008{2009	unsigned long start_time = jiffies;2010	long wrote = 0;2011 2012	while (!list_empty(&wb->b_io)) {2013		struct inode *inode = wb_inode(wb->b_io.prev);2014		struct super_block *sb = inode->i_sb;2015 2016		if (!super_trylock_shared(sb)) {2017			/*2018			 * super_trylock_shared() may fail consistently due to2019			 * s_umount being grabbed by someone else. Don't use2020			 * requeue_io() to avoid busy retrying the inode/sb.2021			 */2022			redirty_tail(inode, wb);2023			continue;2024		}2025		wrote += writeback_sb_inodes(sb, wb, work);2026		up_read(&sb->s_umount);2027 2028		/* refer to the same tests at the end of writeback_sb_inodes */2029		if (wrote) {2030			if (time_is_before_jiffies(start_time + HZ / 10UL))2031				break;2032			if (work->nr_pages <= 0)2033				break;2034		}2035	}2036	/* Leave any unwritten inodes on b_io */2037	return wrote;2038}2039 2040static HWJS_SUSPENDS long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,2041				enum wb_reason reason)2042{2043	struct wb_writeback_work work = {2044		.nr_pages	= nr_pages,2045		.sync_mode	= WB_SYNC_NONE,2046		.range_cyclic	= 1,2047		.reason		= reason,2048	};2049	struct blk_plug plug;2050 2051	blk_start_plug(&plug);2052	spin_lock(&wb->list_lock);2053	if (list_empty(&wb->b_io))2054		queue_io(wb, &work, jiffies);2055	__writeback_inodes_wb(wb, &work);2056	spin_unlock(&wb->list_lock);2057	blk_finish_plug(&plug);2058 2059	return nr_pages - work.nr_pages;2060}2061 2062/*2063 * Explicit flushing or periodic writeback of "old" data.2064 *2065 * Define "old": the first time one of an inode's pages is dirtied, we mark the2066 * dirtying-time in the inode's address_space.  So this periodic writeback code2067 * just walks the superblock inode list, writing back any inodes which are2068 * older than a specific point in time.2069 *2070 * Try to run once per dirty_writeback_interval.  But if a writeback event2071 * takes longer than a dirty_writeback_interval interval, then leave a2072 * one-second gap.2073 *2074 * dirtied_before takes precedence over nr_to_write.  So we'll only write back2075 * all dirty pages if they are all attached to "old" mappings.2076 */2077static HWJS_SUSPENDS long wb_writeback(struct bdi_writeback *wb,2078			 struct wb_writeback_work *work)2079{2080	long nr_pages = work->nr_pages;2081	unsigned long dirtied_before = jiffies;2082	struct inode *inode;2083	long progress;2084	struct blk_plug plug;2085	bool queued = false;2086 2087	blk_start_plug(&plug);2088	for (;;) {2089		/*2090		 * Stop writeback when nr_pages has been consumed2091		 */2092		if (work->nr_pages <= 0)2093			break;2094 2095		/*2096		 * Background writeout and kupdate-style writeback may2097		 * run forever. Stop them if there is other work to do2098		 * so that e.g. sync can proceed. They'll be restarted2099		 * after the other works are all done.2100		 */2101		if ((work->for_background || work->for_kupdate) &&2102		    !list_empty(&wb->work_list))2103			break;2104 2105		/*2106		 * For background writeout, stop when we are below the2107		 * background dirty threshold2108		 */2109		if (work->for_background && !wb_over_bg_thresh(wb))2110			break;2111 2112 2113		spin_lock(&wb->list_lock);2114 2115		trace_writeback_start(wb, work);2116		if (list_empty(&wb->b_io)) {2117			/*2118			 * Kupdate and background works are special and we want2119			 * to include all inodes that need writing. Livelock2120			 * avoidance is handled by these works yielding to any2121			 * other work so we are safe.2122			 */2123			if (work->for_kupdate) {2124				dirtied_before = jiffies -2125					msecs_to_jiffies(dirty_expire_interval *2126							 10);2127			} else if (work->for_background)2128				dirtied_before = jiffies;2129 2130			queue_io(wb, work, dirtied_before);2131			queued = true;2132		}2133		if (work->sb)2134			progress = writeback_sb_inodes(work->sb, wb, work);2135		else2136			progress = __writeback_inodes_wb(wb, work);2137		trace_writeback_written(wb, work);2138 2139		/*2140		 * Did we write something? Try for more2141		 *2142		 * Dirty inodes are moved to b_io for writeback in batches.2143		 * The completion of the current batch does not necessarily2144		 * mean the overall work is done. So we keep looping as long2145		 * as made some progress on cleaning pages or inodes.2146		 */2147		if (progress || !queued) {2148			spin_unlock(&wb->list_lock);2149			continue;2150		}2151 2152		/*2153		 * No more inodes for IO, bail2154		 */2155		if (list_empty(&wb->b_more_io)) {2156			spin_unlock(&wb->list_lock);2157			break;2158		}2159 2160		/*2161		 * Nothing written. Wait for some inode to2162		 * become available for writeback. Otherwise2163		 * we'll just busyloop.2164		 */2165		trace_writeback_wait(wb, work);2166		inode = wb_inode(wb->b_more_io.prev);2167		spin_lock(&inode->i_lock);2168		spin_unlock(&wb->list_lock);2169		/* This function drops i_lock... */2170		inode_sleep_on_writeback(inode);2171	}2172	blk_finish_plug(&plug);2173 2174	return nr_pages - work->nr_pages;2175}2176 2177/*2178 * Return the next wb_writeback_work struct that hasn't been processed yet.2179 */2180static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)2181{2182	struct wb_writeback_work *work = NULL;2183 2184	spin_lock_irq(&wb->work_lock);2185	if (!list_empty(&wb->work_list)) {2186		work = list_entry(wb->work_list.next,2187				  struct wb_writeback_work, list);2188		list_del_init(&work->list);2189	}2190	spin_unlock_irq(&wb->work_lock);2191	return work;2192}2193 2194static HWJS_SUSPENDS long wb_check_background_flush(struct bdi_writeback *wb)2195{2196	if (wb_over_bg_thresh(wb)) {2197 2198		struct wb_writeback_work work = {2199			.nr_pages	= LONG_MAX,2200			.sync_mode	= WB_SYNC_NONE,2201			.for_background	= 1,2202			.range_cyclic	= 1,2203			.reason		= WB_REASON_BACKGROUND,2204		};2205 2206		return wb_writeback(wb, &work);2207	}2208 2209	return 0;2210}2211 2212static HWJS_SUSPENDS long wb_check_old_data_flush(struct bdi_writeback *wb)2213{2214	unsigned long expired;2215	long nr_pages;2216 2217	/*2218	 * When set to zero, disable periodic writeback2219	 */2220	if (!dirty_writeback_interval)2221		return 0;2222 2223	expired = wb->last_old_flush +2224			msecs_to_jiffies(dirty_writeback_interval * 10);2225	if (time_before(jiffies, expired))2226		return 0;2227 2228	wb->last_old_flush = jiffies;2229	nr_pages = get_nr_dirty_pages();2230 2231	if (nr_pages) {2232		struct wb_writeback_work work = {2233			.nr_pages	= nr_pages,2234			.sync_mode	= WB_SYNC_NONE,2235			.for_kupdate	= 1,2236			.range_cyclic	= 1,2237			.reason		= WB_REASON_PERIODIC,2238		};2239 2240		return wb_writeback(wb, &work);2241	}2242 2243	return 0;2244}2245 2246static HWJS_SUSPENDS long wb_check_start_all(struct bdi_writeback *wb)2247{2248	long nr_pages;2249 2250	if (!test_bit(WB_start_all, &wb->state))2251		return 0;2252 2253	nr_pages = get_nr_dirty_pages();2254	if (nr_pages) {2255		struct wb_writeback_work work = {2256			.nr_pages	= wb_split_bdi_pages(wb, nr_pages),2257			.sync_mode	= WB_SYNC_NONE,2258			.range_cyclic	= 1,2259			.reason		= wb->start_all_reason,2260		};2261 2262		nr_pages = wb_writeback(wb, &work);2263	}2264 2265	clear_bit(WB_start_all, &wb->state);2266	return nr_pages;2267}2268 2269 2270/*2271 * Retrieve work items and do the writeback they describe2272 */2273static HWJS_SUSPENDS long wb_do_writeback(struct bdi_writeback *wb)2274{2275	struct wb_writeback_work *work;2276	long wrote = 0;2277 2278	set_bit(WB_writeback_running, &wb->state);2279	while ((work = get_next_work_item(wb)) != NULL) {2280		trace_writeback_exec(wb, work);2281		wrote += wb_writeback(wb, work);2282		finish_writeback_work(work);2283	}2284 2285	/*2286	 * Check for a flush-everything request2287	 */2288	wrote += wb_check_start_all(wb);2289 2290	/*2291	 * Check for periodic writeback, kupdated() style2292	 */2293	wrote += wb_check_old_data_flush(wb);2294	wrote += wb_check_background_flush(wb);2295	clear_bit(WB_writeback_running, &wb->state);2296 2297	return wrote;2298}2299 2300/*2301 * Handle writeback of dirty data for the device backed by this bdi. Also2302 * reschedules periodically and does kupdated style flushing.2303 */2304void wb_workfn(struct work_struct *work)2305{2306	struct bdi_writeback *wb = container_of(to_delayed_work(work),2307						struct bdi_writeback, dwork);2308	long pages_written;2309 2310	set_worker_desc("flush-%s", bdi_dev_name(wb->bdi));2311 2312	if (likely(!current_is_workqueue_rescuer() ||2313		   !test_bit(WB_registered, &wb->state))) {2314		/*2315		 * The normal path.  Keep writing back @wb until its2316		 * work_list is empty.  Note that this path is also taken2317		 * if @wb is shutting down even when we're running off the2318		 * rescuer as work_list needs to be drained.2319		 */2320		do {2321			pages_written = wb_do_writeback(wb);2322			trace_writeback_pages_written(pages_written);2323		} while (!list_empty(&wb->work_list));2324	} else {2325		/*2326		 * bdi_wq can't get enough workers and we're running off2327		 * the emergency worker.  Don't hog it.  Hopefully, 1024 is2328		 * enough for efficient IO.2329		 */2330		pages_written = writeback_inodes_wb(wb, 1024,2331						    WB_REASON_FORKER_THREAD);2332		trace_writeback_pages_written(pages_written);2333	}2334 2335	if (!list_empty(&wb->work_list))2336		wb_wakeup(wb);2337	else if (wb_has_dirty_io(wb) && dirty_writeback_interval)2338		wb_wakeup_delayed(wb);2339}2340 2341/*2342 * Start writeback of all dirty pages on this bdi.2343 */2344static HWJS_SUSPENDS void __wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,2345					 enum wb_reason reason)2346{2347	struct bdi_writeback *wb;2348 2349	if (!bdi_has_dirty_io(bdi))2350		return;2351 2352	list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)2353		wb_start_writeback(wb, reason);2354}2355 2356void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,2357				enum wb_reason reason)2358{2359	rcu_read_lock();2360	__wakeup_flusher_threads_bdi(bdi, reason);2361	rcu_read_unlock();2362}2363 2364/*2365 * Wakeup the flusher threads to start writeback of all currently dirty pages2366 */2367void wakeup_flusher_threads(enum wb_reason reason)2368{2369	struct backing_dev_info *bdi;2370 2371	/*2372	 * If we are expecting writeback progress we must submit plugged IO.2373	 */2374	blk_flush_plug(current->plug, true);2375 2376	rcu_read_lock();2377	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)2378		__wakeup_flusher_threads_bdi(bdi, reason);2379	rcu_read_unlock();2380}2381 2382/*2383 * Wake up bdi's periodically to make sure dirtytime inodes gets2384 * written back periodically.  We deliberately do *not* check the2385 * b_dirtytime list in wb_has_dirty_io(), since this would cause the2386 * kernel to be constantly waking up once there are any dirtytime2387 * inodes on the system.  So instead we define a separate delayed work2388 * function which gets called much more rarely.  (By default, only2389 * once every 12 hours.)2390 *2391 * If there is any other write activity going on in the file system,2392 * this function won't be necessary.  But if the only thing that has2393 * happened on the file system is a dirtytime inode caused by an atime2394 * update, we need this infrastructure below to make sure that inode2395 * eventually gets pushed out to disk.2396 */2397static HWJS_SUSPENDS void wakeup_dirtytime_writeback(struct work_struct *w);2398static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);2399 2400static HWJS_SUSPENDS void wakeup_dirtytime_writeback(struct work_struct *w)2401{2402	struct backing_dev_info *bdi;2403 2404	rcu_read_lock();2405	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {2406		struct bdi_writeback *wb;2407 2408		list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)2409			if (!list_empty(&wb->b_dirty_time))2410				wb_wakeup(wb);2411	}2412	rcu_read_unlock();2413	schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);2414}2415 2416static HWJS_SUSPENDS int __init start_dirtytime_writeback(void)2417{2418	schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);2419	return 0;2420}2421__initcall(start_dirtytime_writeback);2422 2423int dirtytime_interval_handler(const struct ctl_table *table, int write,2424			       void *buffer, size_t *lenp, loff_t *ppos)2425{2426	int ret;2427 2428	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);2429	if (ret == 0 && write)2430		mod_delayed_work(system_wq, &dirtytime_work, 0);2431	return ret;2432}2433 2434/**2435 * __mark_inode_dirty -	internal function to mark an inode dirty2436 *2437 * @inode: inode to mark2438 * @flags: what kind of dirty, e.g. I_DIRTY_SYNC.  This can be a combination of2439 *	   multiple I_DIRTY_* flags, except that I_DIRTY_TIME can't be combined2440 *	   with I_DIRTY_PAGES.2441 *2442 * Mark an inode as dirty.  We notify the filesystem, then update the inode's2443 * dirty flags.  Then, if needed we add the inode to the appropriate dirty list.2444 *2445 * Most callers should use mark_inode_dirty() or mark_inode_dirty_sync()2446 * instead of calling this directly.2447 *2448 * CAREFUL!  We only add the inode to the dirty list if it is hashed or if it2449 * refers to a blockdev.  Unhashed inodes will never be added to the dirty list2450 * even if they are later hashed, as they will have been marked dirty already.2451 *2452 * In short, ensure you hash any inodes _before_ you start marking them dirty.2453 *2454 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of2455 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of2456 * the kernel-internal blockdev inode represents the dirtying time of the2457 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use2458 * page->mapping->host, so the page-dirtying time is recorded in the internal2459 * blockdev inode.2460 */2461void __mark_inode_dirty(struct inode *inode, int flags)2462{2463	struct super_block *sb = inode->i_sb;2464	int dirtytime = 0;2465	struct bdi_writeback *wb = NULL;2466 2467	trace_writeback_mark_inode_dirty(inode, flags);2468 2469	if (flags & I_DIRTY_INODE) {2470		/*2471		 * Inode timestamp update will piggback on this dirtying.2472		 * We tell ->dirty_inode callback that timestamps need to2473		 * be updated by setting I_DIRTY_TIME in flags.2474		 */2475		if (inode->i_state & I_DIRTY_TIME) {2476			spin_lock(&inode->i_lock);2477			if (inode->i_state & I_DIRTY_TIME) {2478				inode->i_state &= ~I_DIRTY_TIME;2479				flags |= I_DIRTY_TIME;2480			}2481			spin_unlock(&inode->i_lock);2482		}2483 2484		/*2485		 * Notify the filesystem about the inode being dirtied, so that2486		 * (if needed) it can update on-disk fields and journal the2487		 * inode.  This is only needed when the inode itself is being2488		 * dirtied now.  I.e. it's only needed for I_DIRTY_INODE, not2489		 * for just I_DIRTY_PAGES or I_DIRTY_TIME.2490		 */2491		trace_writeback_dirty_inode_start(inode, flags);2492		if (sb->s_op->dirty_inode)2493			sb->s_op->dirty_inode(inode,2494				flags & (I_DIRTY_INODE | I_DIRTY_TIME));2495		trace_writeback_dirty_inode(inode, flags);2496 2497		/* I_DIRTY_INODE supersedes I_DIRTY_TIME. */2498		flags &= ~I_DIRTY_TIME;2499	} else {2500		/*2501		 * Else it's either I_DIRTY_PAGES, I_DIRTY_TIME, or nothing.2502		 * (We don't support setting both I_DIRTY_PAGES and I_DIRTY_TIME2503		 * in one call to __mark_inode_dirty().)2504		 */2505		dirtytime = flags & I_DIRTY_TIME;2506		WARN_ON_ONCE(dirtytime && flags != I_DIRTY_TIME);2507	}2508 2509	/*2510	 * Paired with smp_mb() in __writeback_single_inode() for the2511	 * following lockless i_state test.  See there for details.2512	 */2513	smp_mb();2514 2515	if ((inode->i_state & flags) == flags)2516		return;2517 2518	spin_lock(&inode->i_lock);2519	if ((inode->i_state & flags) != flags) {2520		const int was_dirty = inode->i_state & I_DIRTY;2521 2522		inode_attach_wb(inode, NULL);2523 2524		inode->i_state |= flags;2525 2526		/*2527		 * Grab inode's wb early because it requires dropping i_lock and we2528		 * need to make sure following checks happen atomically with dirty2529		 * list handling so that we don't move inodes under flush worker's2530		 * hands.2531		 */2532		if (!was_dirty) {2533			wb = locked_inode_to_wb_and_lock_list(inode);2534			spin_lock(&inode->i_lock);2535		}2536 2537		/*2538		 * If the inode is queued for writeback by flush worker, just2539		 * update its dirty state. Once the flush worker is done with2540		 * the inode it will place it on the appropriate superblock2541		 * list, based upon its state.2542		 */2543		if (inode->i_state & I_SYNC_QUEUED)2544			goto out_unlock;2545 2546		/*2547		 * Only add valid (hashed) inodes to the superblock's2548		 * dirty list.  Add blockdev inodes as well.2549		 */2550		if (!S_ISBLK(inode->i_mode)) {2551			if (inode_unhashed(inode))2552				goto out_unlock;2553		}2554		if (inode->i_state & I_FREEING)2555			goto out_unlock;2556 2557		/*2558		 * If the inode was already on b_dirty/b_io/b_more_io, don't2559		 * reposition it (that would break b_dirty time-ordering).2560		 */2561		if (!was_dirty) {2562			struct list_head *dirty_list;2563			bool wakeup_bdi = false;2564 2565			inode->dirtied_when = jiffies;2566			if (dirtytime)2567				inode->dirtied_time_when = jiffies;2568 2569			if (inode->i_state & I_DIRTY)2570				dirty_list = &wb->b_dirty;2571			else2572				dirty_list = &wb->b_dirty_time;2573 2574			wakeup_bdi = inode_io_list_move_locked(inode, wb,2575							       dirty_list);2576 2577			spin_unlock(&wb->list_lock);2578			spin_unlock(&inode->i_lock);2579			trace_writeback_dirty_inode_enqueue(inode);2580 2581			/*2582			 * If this is the first dirty inode for this bdi,2583			 * we have to wake-up the corresponding bdi thread2584			 * to make sure background write-back happens2585			 * later.2586			 */2587			if (wakeup_bdi &&2588			    (wb->bdi->capabilities & BDI_CAP_WRITEBACK))2589				wb_wakeup_delayed(wb);2590			return;2591		}2592	}2593out_unlock:2594	if (wb)2595		spin_unlock(&wb->list_lock);2596	spin_unlock(&inode->i_lock);2597}2598EXPORT_SYMBOL(__mark_inode_dirty);2599 2600/*2601 * The @s_sync_lock is used to serialise concurrent sync operations2602 * to avoid lock contention problems with concurrent wait_sb_inodes() calls.2603 * Concurrent callers will block on the s_sync_lock rather than doing contending2604 * walks. The queueing maintains sync(2) required behaviour as all the IO that2605 * has been issued up to the time this function is enter is guaranteed to be2606 * completed by the time we have gained the lock and waited for all IO that is2607 * in progress regardless of the order callers are granted the lock.2608 */2609static HWJS_SUSPENDS void wait_sb_inodes(struct super_block *sb)2610{2611	LIST_HEAD(sync_list);2612 2613	/*2614	 * We need to be protected against the filesystem going from2615	 * r/o to r/w or vice versa.2616	 */2617	WARN_ON(!rwsem_is_locked(&sb->s_umount));2618 2619	mutex_lock(&sb->s_sync_lock);2620 2621	/*2622	 * Splice the writeback list onto a temporary list to avoid waiting on2623	 * inodes that have started writeback after this point.2624	 *2625	 * Use rcu_read_lock() to keep the inodes around until we have a2626	 * reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as2627	 * the local list because inodes can be dropped from either by writeback2628	 * completion.2629	 */2630	rcu_read_lock();2631	spin_lock_irq(&sb->s_inode_wblist_lock);2632	list_splice_init(&sb->s_inodes_wb, &sync_list);2633 2634	/*2635	 * Data integrity sync. Must wait for all pages under writeback, because2636	 * there may have been pages dirtied before our sync call, but which had2637	 * writeout started before we write it out.  In which case, the inode2638	 * may not be on the dirty list, but we still have to wait for that2639	 * writeout.2640	 */2641	while (!list_empty(&sync_list)) {2642		struct inode *inode = list_first_entry(&sync_list, struct inode,2643						       i_wb_list);2644		struct address_space *mapping = inode->i_mapping;2645 2646		/*2647		 * Move each inode back to the wb list before we drop the lock2648		 * to preserve consistency between i_wb_list and the mapping2649		 * writeback tag. Writeback completion is responsible to remove2650		 * the inode from either list once the writeback tag is cleared.2651		 */2652		list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb);2653 2654		/*2655		 * The mapping can appear untagged while still on-list since we2656		 * do not have the mapping lock. Skip it here, wb completion2657		 * will remove it.2658		 */2659		if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))2660			continue;2661 2662		spin_unlock_irq(&sb->s_inode_wblist_lock);2663 2664		spin_lock(&inode->i_lock);2665		if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) {2666			spin_unlock(&inode->i_lock);2667 2668			spin_lock_irq(&sb->s_inode_wblist_lock);2669			continue;2670		}2671		__iget(inode);2672		spin_unlock(&inode->i_lock);2673		rcu_read_unlock();2674 2675		/*2676		 * We keep the error status of individual mapping so that2677		 * applications can catch the writeback error using fsync(2).2678		 * See filemap_fdatawait_keep_errors() for details.2679		 */2680		filemap_fdatawait_keep_errors(mapping);2681 2682		cond_resched();2683 2684		iput(inode);2685 2686		rcu_read_lock();2687		spin_lock_irq(&sb->s_inode_wblist_lock);2688	}2689	spin_unlock_irq(&sb->s_inode_wblist_lock);2690	rcu_read_unlock();2691	mutex_unlock(&sb->s_sync_lock);2692}2693 2694static HWJS_SUSPENDS void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr,2695				     enum wb_reason reason, bool skip_if_busy)2696{2697	struct backing_dev_info *bdi = sb->s_bdi;2698	DEFINE_WB_COMPLETION(done, bdi);2699	struct wb_writeback_work work = {2700		.sb			= sb,2701		.sync_mode		= WB_SYNC_NONE,2702		.tagged_writepages	= 1,2703		.done			= &done,2704		.nr_pages		= nr,2705		.reason			= reason,2706	};2707 2708	if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)2709		return;2710	WARN_ON(!rwsem_is_locked(&sb->s_umount));2711 2712	bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy);2713	wb_wait_for_completion(&done);2714}2715 2716/**2717 * writeback_inodes_sb_nr -	writeback dirty inodes from given super_block2718 * @sb: the superblock2719 * @nr: the number of pages to write2720 * @reason: reason why some writeback work initiated2721 *2722 * Start writeback on some inodes on this super_block. No guarantees are made2723 * on how many (if any) will be written, and this function does not wait2724 * for IO completion of submitted IO.2725 */2726void writeback_inodes_sb_nr(struct super_block *sb,2727			    unsigned long nr,2728			    enum wb_reason reason)2729{2730	__writeback_inodes_sb_nr(sb, nr, reason, false);2731}2732EXPORT_SYMBOL(writeback_inodes_sb_nr);2733 2734/**2735 * writeback_inodes_sb	-	writeback dirty inodes from given super_block2736 * @sb: the superblock2737 * @reason: reason why some writeback work was initiated2738 *2739 * Start writeback on some inodes on this super_block. No guarantees are made2740 * on how many (if any) will be written, and this function does not wait2741 * for IO completion of submitted IO.2742 */2743void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)2744{2745	writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);2746}2747EXPORT_SYMBOL(writeback_inodes_sb);2748 2749/**2750 * try_to_writeback_inodes_sb - try to start writeback if none underway2751 * @sb: the superblock2752 * @reason: reason why some writeback work was initiated2753 *2754 * Invoke __writeback_inodes_sb_nr if no writeback is currently underway.2755 */2756void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)2757{2758	if (!down_read_trylock(&sb->s_umount))2759		return;2760 2761	__writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason, true);2762	up_read(&sb->s_umount);2763}2764EXPORT_SYMBOL(try_to_writeback_inodes_sb);2765 2766/**2767 * sync_inodes_sb	-	sync sb inode pages2768 * @sb: the superblock2769 *2770 * This function writes and waits on any dirty inode belonging to this2771 * super_block.2772 */2773void sync_inodes_sb(struct super_block *sb)2774{2775	struct backing_dev_info *bdi = sb->s_bdi;2776	DEFINE_WB_COMPLETION(done, bdi);2777	struct wb_writeback_work work = {2778		.sb		= sb,2779		.sync_mode	= WB_SYNC_ALL,2780		.nr_pages	= LONG_MAX,2781		.range_cyclic	= 0,2782		.done		= &done,2783		.reason		= WB_REASON_SYNC,2784		.for_sync	= 1,2785	};2786 2787	/*2788	 * Can't skip on !bdi_has_dirty() because we should wait for !dirty2789	 * inodes under writeback and I_DIRTY_TIME inodes ignored by2790	 * bdi_has_dirty() need to be written out too.2791	 */2792	if (bdi == &noop_backing_dev_info)2793		return;2794	WARN_ON(!rwsem_is_locked(&sb->s_umount));2795 2796	/* protect against inode wb switch, see inode_switch_wbs_work_fn() */2797	bdi_down_write_wb_switch_rwsem(bdi);2798	bdi_split_work_to_wbs(bdi, &work, false);2799	wb_wait_for_completion(&done);2800	bdi_up_write_wb_switch_rwsem(bdi);2801 2802	wait_sb_inodes(sb);2803}2804EXPORT_SYMBOL(sync_inodes_sb);2805 2806/**2807 * write_inode_now	-	write an inode to disk2808 * @inode: inode to write to disk2809 * @sync: whether the write should be synchronous or not2810 *2811 * This function commits an inode to disk immediately if it is dirty. This is2812 * primarily needed by knfsd.2813 *2814 * The caller must either have a ref on the inode or must have set I_WILL_FREE.2815 */2816int write_inode_now(struct inode *inode, int sync)2817{2818	struct writeback_control wbc = {2819		.nr_to_write = LONG_MAX,2820		.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,2821		.range_start = 0,2822		.range_end = LLONG_MAX,2823	};2824 2825	if (!mapping_can_writeback(inode->i_mapping))2826		wbc.nr_to_write = 0;2827 2828	might_sleep();2829	return writeback_single_inode(inode, &wbc);2830}2831EXPORT_SYMBOL(write_inode_now);2832 2833/**2834 * sync_inode_metadata - write an inode to disk2835 * @inode: the inode to sync2836 * @wait: wait for I/O to complete.2837 *2838 * Write an inode to disk and adjust its dirty state after completion.2839 *2840 * Note: only writes the actual inode, no associated data or other metadata.2841 */2842int sync_inode_metadata(struct inode *inode, int wait)2843{2844	struct writeback_control wbc = {2845		.sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,2846		.nr_to_write = 0, /* metadata-only */2847	};2848 2849	return writeback_single_inode(inode, &wbc);2850}2851EXPORT_SYMBOL(sync_inode_metadata);2852