brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/*3 *  linux/fs/ext4/inode.c4 *5 * Copyright (C) 1992, 1993, 1994, 19956 * Remy Card (card@masi.ibp.fr)7 * Laboratoire MASI - Institut Blaise Pascal8 * Universite Pierre et Marie Curie (Paris VI)9 *10 *  from11 *12 *  linux/fs/minix/inode.c13 *14 *  Copyright (C) 1991, 1992  Linus Torvalds15 *16 *  64-bit file support on 64-bit platforms by Jakub Jelinek17 *	(jj@sunsite.ms.mff.cuni.cz)18 *19 *  Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 200020 */21 22#include <linux/fs.h>23#include <linux/mount.h>24#include <linux/time.h>25#include <linux/highuid.h>26#include <linux/pagemap.h>27#include <linux/dax.h>28#include <linux/quotaops.h>29#include <linux/string.h>30#include <linux/buffer_head.h>31#include <linux/writeback.h>32#include <linux/pagevec.h>33#include <linux/mpage.h>34#include <linux/namei.h>35#include <linux/uio.h>36#include <linux/bio.h>37#include <linux/workqueue.h>38#include <linux/kernel.h>39#include <linux/printk.h>40#include <linux/slab.h>41#include <linux/bitops.h>42#include <linux/iomap.h>43#include <linux/iversion.h>44 45#include "ext4_jbd2.h"46#include "xattr.h"47#include "acl.h"48#include "truncate.h"49 50#include <trace/events/ext4.h>51 52static void ext4_journalled_zero_new_buffers(handle_t *handle,53					    struct inode *inode,54					    struct folio *folio,55					    unsigned from, unsigned to);56 57static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,58			      struct ext4_inode_info *ei)59{60	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);61	__u32 csum;62	__u16 dummy_csum = 0;63	int offset = offsetof(struct ext4_inode, i_checksum_lo);64	unsigned int csum_size = sizeof(dummy_csum);65 66	csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);67	csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);68	offset += csum_size;69	csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,70			   EXT4_GOOD_OLD_INODE_SIZE - offset);71 72	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {73		offset = offsetof(struct ext4_inode, i_checksum_hi);74		csum = ext4_chksum(sbi, csum, (__u8 *)raw +75				   EXT4_GOOD_OLD_INODE_SIZE,76				   offset - EXT4_GOOD_OLD_INODE_SIZE);77		if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {78			csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,79					   csum_size);80			offset += csum_size;81		}82		csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,83				   EXT4_INODE_SIZE(inode->i_sb) - offset);84	}85 86	return csum;87}88 89static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,90				  struct ext4_inode_info *ei)91{92	__u32 provided, calculated;93 94	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=95	    cpu_to_le32(EXT4_OS_LINUX) ||96	    !ext4_has_metadata_csum(inode->i_sb))97		return 1;98 99	provided = le16_to_cpu(raw->i_checksum_lo);100	calculated = ext4_inode_csum(inode, raw, ei);101	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&102	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))103		provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;104	else105		calculated &= 0xFFFF;106 107	return provided == calculated;108}109 110void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,111			 struct ext4_inode_info *ei)112{113	__u32 csum;114 115	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=116	    cpu_to_le32(EXT4_OS_LINUX) ||117	    !ext4_has_metadata_csum(inode->i_sb))118		return;119 120	csum = ext4_inode_csum(inode, raw, ei);121	raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);122	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&123	    EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))124		raw->i_checksum_hi = cpu_to_le16(csum >> 16);125}126 127static inline int ext4_begin_ordered_truncate(struct inode *inode,128					      loff_t new_size)129{130	trace_ext4_begin_ordered_truncate(inode, new_size);131	/*132	 * If jinode is zero, then we never opened the file for133	 * writing, so there's no need to call134	 * jbd2_journal_begin_ordered_truncate() since there's no135	 * outstanding writes we need to flush.136	 */137	if (!EXT4_I(inode)->jinode)138		return 0;139	return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),140						   EXT4_I(inode)->jinode,141						   new_size);142}143 144static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,145				  int pextents);146 147/*148 * Test whether an inode is a fast symlink.149 * A fast symlink has its symlink data stored in ext4_inode_info->i_data.150 */151int ext4_inode_is_fast_symlink(struct inode *inode)152{153	if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {154		int ea_blocks = EXT4_I(inode)->i_file_acl ?155				EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;156 157		if (ext4_has_inline_data(inode))158			return 0;159 160		return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);161	}162	return S_ISLNK(inode->i_mode) && inode->i_size &&163	       (inode->i_size < EXT4_N_BLOCKS * 4);164}165 166/*167 * Called at the last iput() if i_nlink is zero.168 */169void ext4_evict_inode(struct inode *inode)170{171	handle_t *handle;172	int err;173	/*174	 * Credits for final inode cleanup and freeing:175	 * sb + inode (ext4_orphan_del()), block bitmap, group descriptor176	 * (xattr block freeing), bitmap, group descriptor (inode freeing)177	 */178	int extra_credits = 6;179	struct ext4_xattr_inode_array *ea_inode_array = NULL;180	bool freeze_protected = false;181 182	trace_ext4_evict_inode(inode);183 184	if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)185		ext4_evict_ea_inode(inode);186	if (inode->i_nlink) {187		truncate_inode_pages_final(&inode->i_data);188 189		goto no_delete;190	}191 192	if (is_bad_inode(inode))193		goto no_delete;194	dquot_initialize(inode);195 196	if (ext4_should_order_data(inode))197		ext4_begin_ordered_truncate(inode, 0);198	truncate_inode_pages_final(&inode->i_data);199 200	/*201	 * For inodes with journalled data, transaction commit could have202	 * dirtied the inode. And for inodes with dioread_nolock, unwritten203	 * extents converting worker could merge extents and also have dirtied204	 * the inode. Flush worker is ignoring it because of I_FREEING flag but205	 * we still need to remove the inode from the writeback lists.206	 */207	if (!list_empty_careful(&inode->i_io_list))208		inode_io_list_del(inode);209 210	/*211	 * Protect us against freezing - iput() caller didn't have to have any212	 * protection against it. When we are in a running transaction though,213	 * we are already protected against freezing and we cannot grab further214	 * protection due to lock ordering constraints.215	 */216	if (!ext4_journal_current_handle()) {217		sb_start_intwrite(inode->i_sb);218		freeze_protected = true;219	}220 221	if (!IS_NOQUOTA(inode))222		extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);223 224	/*225	 * Block bitmap, group descriptor, and inode are accounted in both226	 * ext4_blocks_for_truncate() and extra_credits. So subtract 3.227	 */228	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,229			 ext4_blocks_for_truncate(inode) + extra_credits - 3);230	if (IS_ERR(handle)) {231		ext4_std_error(inode->i_sb, PTR_ERR(handle));232		/*233		 * If we're going to skip the normal cleanup, we still need to234		 * make sure that the in-core orphan linked list is properly235		 * cleaned up.236		 */237		ext4_orphan_del(NULL, inode);238		if (freeze_protected)239			sb_end_intwrite(inode->i_sb);240		goto no_delete;241	}242 243	if (IS_SYNC(inode))244		ext4_handle_sync(handle);245 246	/*247	 * Set inode->i_size to 0 before calling ext4_truncate(). We need248	 * special handling of symlinks here because i_size is used to249	 * determine whether ext4_inode_info->i_data contains symlink data or250	 * block mappings. Setting i_size to 0 will remove its fast symlink251	 * status. Erase i_data so that it becomes a valid empty block map.252	 */253	if (ext4_inode_is_fast_symlink(inode))254		memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));255	inode->i_size = 0;256	err = ext4_mark_inode_dirty(handle, inode);257	if (err) {258		ext4_warning(inode->i_sb,259			     "couldn't mark inode dirty (err %d)", err);260		goto stop_handle;261	}262	if (inode->i_blocks) {263		err = ext4_truncate(inode);264		if (err) {265			ext4_error_err(inode->i_sb, -err,266				       "couldn't truncate inode %lu (err %d)",267				       inode->i_ino, err);268			goto stop_handle;269		}270	}271 272	/* Remove xattr references. */273	err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,274				      extra_credits);275	if (err) {276		ext4_warning(inode->i_sb, "xattr delete (err %d)", err);277stop_handle:278		ext4_journal_stop(handle);279		ext4_orphan_del(NULL, inode);280		if (freeze_protected)281			sb_end_intwrite(inode->i_sb);282		ext4_xattr_inode_array_free(ea_inode_array);283		goto no_delete;284	}285 286	/*287	 * Kill off the orphan record which ext4_truncate created.288	 * AKPM: I think this can be inside the above `if'.289	 * Note that ext4_orphan_del() has to be able to cope with the290	 * deletion of a non-existent orphan - this is because we don't291	 * know if ext4_truncate() actually created an orphan record.292	 * (Well, we could do this if we need to, but heck - it works)293	 */294	ext4_orphan_del(handle, inode);295	EXT4_I(inode)->i_dtime	= (__u32)ktime_get_real_seconds();296 297	/*298	 * One subtle ordering requirement: if anything has gone wrong299	 * (transaction abort, IO errors, whatever), then we can still300	 * do these next steps (the fs will already have been marked as301	 * having errors), but we can't free the inode if the mark_dirty302	 * fails.303	 */304	if (ext4_mark_inode_dirty(handle, inode))305		/* If that failed, just do the required in-core inode clear. */306		ext4_clear_inode(inode);307	else308		ext4_free_inode(handle, inode);309	ext4_journal_stop(handle);310	if (freeze_protected)311		sb_end_intwrite(inode->i_sb);312	ext4_xattr_inode_array_free(ea_inode_array);313	return;314no_delete:315	/*316	 * Check out some where else accidentally dirty the evicting inode,317	 * which may probably cause inode use-after-free issues later.318	 */319	WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list));320 321	if (!list_empty(&EXT4_I(inode)->i_fc_list))322		ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);323	ext4_clear_inode(inode);	/* We must guarantee clearing of inode... */324}325 326#ifdef CONFIG_QUOTA327qsize_t *ext4_get_reserved_space(struct inode *inode)328{329	return &EXT4_I(inode)->i_reserved_quota;330}331#endif332 333/*334 * Called with i_data_sem down, which is important since we can call335 * ext4_discard_preallocations() from here.336 */337void ext4_da_update_reserve_space(struct inode *inode,338					int used, int quota_claim)339{340	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);341	struct ext4_inode_info *ei = EXT4_I(inode);342 343	spin_lock(&ei->i_block_reservation_lock);344	trace_ext4_da_update_reserve_space(inode, used, quota_claim);345	if (unlikely(used > ei->i_reserved_data_blocks)) {346		ext4_warning(inode->i_sb, "%s: ino %lu, used %d "347			 "with only %d reserved data blocks",348			 __func__, inode->i_ino, used,349			 ei->i_reserved_data_blocks);350		WARN_ON(1);351		used = ei->i_reserved_data_blocks;352	}353 354	/* Update per-inode reservations */355	ei->i_reserved_data_blocks -= used;356	percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);357 358	spin_unlock(&ei->i_block_reservation_lock);359 360	/* Update quota subsystem for data blocks */361	if (quota_claim)362		dquot_claim_block(inode, EXT4_C2B(sbi, used));363	else {364		/*365		 * We did fallocate with an offset that is already delayed366		 * allocated. So on delayed allocated writeback we should367		 * not re-claim the quota for fallocated blocks.368		 */369		dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));370	}371 372	/*373	 * If we have done all the pending block allocations and if374	 * there aren't any writers on the inode, we can discard the375	 * inode's preallocations.376	 */377	if ((ei->i_reserved_data_blocks == 0) &&378	    !inode_is_open_for_write(inode))379		ext4_discard_preallocations(inode);380}381 382static int __check_block_validity(struct inode *inode, const char *func,383				unsigned int line,384				struct ext4_map_blocks *map)385{386	if (ext4_has_feature_journal(inode->i_sb) &&387	    (inode->i_ino ==388	     le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))389		return 0;390	if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {391		ext4_error_inode(inode, func, line, map->m_pblk,392				 "lblock %lu mapped to illegal pblock %llu "393				 "(length %d)", (unsigned long) map->m_lblk,394				 map->m_pblk, map->m_len);395		return -EFSCORRUPTED;396	}397	return 0;398}399 400int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,401		       ext4_lblk_t len)402{403	int ret;404 405	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))406		return fscrypt_zeroout_range(inode, lblk, pblk, len);407 408	ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);409	if (ret > 0)410		ret = 0;411 412	return ret;413}414 415#define check_block_validity(inode, map)	\416	__check_block_validity((inode), __func__, __LINE__, (map))417 418#ifdef ES_AGGRESSIVE_TEST419static void ext4_map_blocks_es_recheck(handle_t *handle,420				       struct inode *inode,421				       struct ext4_map_blocks *es_map,422				       struct ext4_map_blocks *map,423				       int flags)424{425	int retval;426 427	map->m_flags = 0;428	/*429	 * There is a race window that the result is not the same.430	 * e.g. xfstests #223 when dioread_nolock enables.  The reason431	 * is that we lookup a block mapping in extent status tree with432	 * out taking i_data_sem.  So at the time the unwritten extent433	 * could be converted.434	 */435	down_read(&EXT4_I(inode)->i_data_sem);436	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {437		retval = ext4_ext_map_blocks(handle, inode, map, 0);438	} else {439		retval = ext4_ind_map_blocks(handle, inode, map, 0);440	}441	up_read((&EXT4_I(inode)->i_data_sem));442 443	/*444	 * We don't check m_len because extent will be collpased in status445	 * tree.  So the m_len might not equal.446	 */447	if (es_map->m_lblk != map->m_lblk ||448	    es_map->m_flags != map->m_flags ||449	    es_map->m_pblk != map->m_pblk) {450		printk("ES cache assertion failed for inode: %lu "451		       "es_cached ex [%d/%d/%llu/%x] != "452		       "found ex [%d/%d/%llu/%x] retval %d flags %x\n",453		       inode->i_ino, es_map->m_lblk, es_map->m_len,454		       es_map->m_pblk, es_map->m_flags, map->m_lblk,455		       map->m_len, map->m_pblk, map->m_flags,456		       retval, flags);457	}458}459#endif /* ES_AGGRESSIVE_TEST */460 461static int ext4_map_query_blocks(handle_t *handle, struct inode *inode,462				 struct ext4_map_blocks *map)463{464	unsigned int status;465	int retval;466 467	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))468		retval = ext4_ext_map_blocks(handle, inode, map, 0);469	else470		retval = ext4_ind_map_blocks(handle, inode, map, 0);471 472	if (retval <= 0)473		return retval;474 475	if (unlikely(retval != map->m_len)) {476		ext4_warning(inode->i_sb,477			     "ES len assertion failed for inode "478			     "%lu: retval %d != map->m_len %d",479			     inode->i_ino, retval, map->m_len);480		WARN_ON(1);481	}482 483	status = map->m_flags & EXT4_MAP_UNWRITTEN ?484			EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;485	ext4_es_insert_extent(inode, map->m_lblk, map->m_len,486			      map->m_pblk, status, 0);487	return retval;488}489 490static int ext4_map_create_blocks(handle_t *handle, struct inode *inode,491				  struct ext4_map_blocks *map, int flags)492{493	struct extent_status es;494	unsigned int status;495	int err, retval = 0;496 497	/*498	 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE499	 * indicates that the blocks and quotas has already been500	 * checked when the data was copied into the page cache.501	 */502	if (map->m_flags & EXT4_MAP_DELAYED)503		flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;504 505	/*506	 * Here we clear m_flags because after allocating an new extent,507	 * it will be set again.508	 */509	map->m_flags &= ~EXT4_MAP_FLAGS;510 511	/*512	 * We need to check for EXT4 here because migrate could have513	 * changed the inode type in between.514	 */515	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {516		retval = ext4_ext_map_blocks(handle, inode, map, flags);517	} else {518		retval = ext4_ind_map_blocks(handle, inode, map, flags);519 520		/*521		 * We allocated new blocks which will result in i_data's522		 * format changing. Force the migrate to fail by clearing523		 * migrate flags.524		 */525		if (retval > 0 && map->m_flags & EXT4_MAP_NEW)526			ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);527	}528	if (retval <= 0)529		return retval;530 531	if (unlikely(retval != map->m_len)) {532		ext4_warning(inode->i_sb,533			     "ES len assertion failed for inode %lu: "534			     "retval %d != map->m_len %d",535			     inode->i_ino, retval, map->m_len);536		WARN_ON(1);537	}538 539	/*540	 * We have to zeroout blocks before inserting them into extent541	 * status tree. Otherwise someone could look them up there and542	 * use them before they are really zeroed. We also have to543	 * unmap metadata before zeroing as otherwise writeback can544	 * overwrite zeros with stale data from block device.545	 */546	if (flags & EXT4_GET_BLOCKS_ZERO &&547	    map->m_flags & EXT4_MAP_MAPPED && map->m_flags & EXT4_MAP_NEW) {548		err = ext4_issue_zeroout(inode, map->m_lblk, map->m_pblk,549					 map->m_len);550		if (err)551			return err;552	}553 554	/*555	 * If the extent has been zeroed out, we don't need to update556	 * extent status tree.557	 */558	if (flags & EXT4_GET_BLOCKS_PRE_IO &&559	    ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {560		if (ext4_es_is_written(&es))561			return retval;562	}563 564	status = map->m_flags & EXT4_MAP_UNWRITTEN ?565			EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;566	ext4_es_insert_extent(inode, map->m_lblk, map->m_len,567			      map->m_pblk, status, flags);568 569	return retval;570}571 572/*573 * The ext4_map_blocks() function tries to look up the requested blocks,574 * and returns if the blocks are already mapped.575 *576 * Otherwise it takes the write lock of the i_data_sem and allocate blocks577 * and store the allocated blocks in the result buffer head and mark it578 * mapped.579 *580 * If file type is extents based, it will call ext4_ext_map_blocks(),581 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping582 * based files583 *584 * On success, it returns the number of blocks being mapped or allocated.585 * If flags doesn't contain EXT4_GET_BLOCKS_CREATE the blocks are586 * pre-allocated and unwritten, the resulting @map is marked as unwritten.587 * If the flags contain EXT4_GET_BLOCKS_CREATE, it will mark @map as mapped.588 *589 * It returns 0 if plain look up failed (blocks have not been allocated), in590 * that case, @map is returned as unmapped but we still do fill map->m_len to591 * indicate the length of a hole starting at map->m_lblk.592 *593 * It returns the error in case of allocation failure.594 */595int ext4_map_blocks(handle_t *handle, struct inode *inode,596		    struct ext4_map_blocks *map, int flags)597{598	struct extent_status es;599	int retval;600	int ret = 0;601#ifdef ES_AGGRESSIVE_TEST602	struct ext4_map_blocks orig_map;603 604	memcpy(&orig_map, map, sizeof(*map));605#endif606 607	map->m_flags = 0;608	ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",609		  flags, map->m_len, (unsigned long) map->m_lblk);610 611	/*612	 * ext4_map_blocks returns an int, and m_len is an unsigned int613	 */614	if (unlikely(map->m_len > INT_MAX))615		map->m_len = INT_MAX;616 617	/* We can handle the block number less than EXT_MAX_BLOCKS */618	if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))619		return -EFSCORRUPTED;620 621	/* Lookup extent status tree firstly */622	if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) &&623	    ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {624		if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {625			map->m_pblk = ext4_es_pblock(&es) +626					map->m_lblk - es.es_lblk;627			map->m_flags |= ext4_es_is_written(&es) ?628					EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;629			retval = es.es_len - (map->m_lblk - es.es_lblk);630			if (retval > map->m_len)631				retval = map->m_len;632			map->m_len = retval;633		} else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {634			map->m_pblk = 0;635			map->m_flags |= ext4_es_is_delayed(&es) ?636					EXT4_MAP_DELAYED : 0;637			retval = es.es_len - (map->m_lblk - es.es_lblk);638			if (retval > map->m_len)639				retval = map->m_len;640			map->m_len = retval;641			retval = 0;642		} else {643			BUG();644		}645 646		if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)647			return retval;648#ifdef ES_AGGRESSIVE_TEST649		ext4_map_blocks_es_recheck(handle, inode, map,650					   &orig_map, flags);651#endif652		goto found;653	}654	/*655	 * In the query cache no-wait mode, nothing we can do more if we656	 * cannot find extent in the cache.657	 */658	if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)659		return 0;660 661	/*662	 * Try to see if we can get the block without requesting a new663	 * file system block.664	 */665	down_read(&EXT4_I(inode)->i_data_sem);666	retval = ext4_map_query_blocks(handle, inode, map);667	up_read((&EXT4_I(inode)->i_data_sem));668 669found:670	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {671		ret = check_block_validity(inode, map);672		if (ret != 0)673			return ret;674	}675 676	/* If it is only a block(s) look up */677	if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)678		return retval;679 680	/*681	 * Returns if the blocks have already allocated682	 *683	 * Note that if blocks have been preallocated684	 * ext4_ext_map_blocks() returns with buffer head unmapped685	 */686	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)687		/*688		 * If we need to convert extent to unwritten689		 * we continue and do the actual work in690		 * ext4_ext_map_blocks()691		 */692		if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))693			return retval;694 695	/*696	 * New blocks allocate and/or writing to unwritten extent697	 * will possibly result in updating i_data, so we take698	 * the write lock of i_data_sem, and call get_block()699	 * with create == 1 flag.700	 */701	down_write(&EXT4_I(inode)->i_data_sem);702	retval = ext4_map_create_blocks(handle, inode, map, flags);703	up_write((&EXT4_I(inode)->i_data_sem));704	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {705		ret = check_block_validity(inode, map);706		if (ret != 0)707			return ret;708 709		/*710		 * Inodes with freshly allocated blocks where contents will be711		 * visible after transaction commit must be on transaction's712		 * ordered data list.713		 */714		if (map->m_flags & EXT4_MAP_NEW &&715		    !(map->m_flags & EXT4_MAP_UNWRITTEN) &&716		    !(flags & EXT4_GET_BLOCKS_ZERO) &&717		    !ext4_is_quota_file(inode) &&718		    ext4_should_order_data(inode)) {719			loff_t start_byte =720				(loff_t)map->m_lblk << inode->i_blkbits;721			loff_t length = (loff_t)map->m_len << inode->i_blkbits;722 723			if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)724				ret = ext4_jbd2_inode_add_wait(handle, inode,725						start_byte, length);726			else727				ret = ext4_jbd2_inode_add_write(handle, inode,728						start_byte, length);729			if (ret)730				return ret;731		}732	}733	if (retval > 0 && (map->m_flags & EXT4_MAP_UNWRITTEN ||734				map->m_flags & EXT4_MAP_MAPPED))735		ext4_fc_track_range(handle, inode, map->m_lblk,736					map->m_lblk + map->m_len - 1);737	if (retval < 0)738		ext_debug(inode, "failed with err %d\n", retval);739	return retval;740}741 742/*743 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages744 * we have to be careful as someone else may be manipulating b_state as well.745 */746static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)747{748	unsigned long old_state;749	unsigned long new_state;750 751	flags &= EXT4_MAP_FLAGS;752 753	/* Dummy buffer_head? Set non-atomically. */754	if (!bh->b_page) {755		bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;756		return;757	}758	/*759	 * Someone else may be modifying b_state. Be careful! This is ugly but760	 * once we get rid of using bh as a container for mapping information761	 * to pass to / from get_block functions, this can go away.762	 */763	old_state = READ_ONCE(bh->b_state);764	do {765		new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;766	} while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state)));767}768 769static int _ext4_get_block(struct inode *inode, sector_t iblock,770			   struct buffer_head *bh, int flags)771{772	struct ext4_map_blocks map;773	int ret = 0;774 775	if (ext4_has_inline_data(inode))776		return -ERANGE;777 778	map.m_lblk = iblock;779	map.m_len = bh->b_size >> inode->i_blkbits;780 781	ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,782			      flags);783	if (ret > 0) {784		map_bh(bh, inode->i_sb, map.m_pblk);785		ext4_update_bh_state(bh, map.m_flags);786		bh->b_size = inode->i_sb->s_blocksize * map.m_len;787		ret = 0;788	} else if (ret == 0) {789		/* hole case, need to fill in bh->b_size */790		bh->b_size = inode->i_sb->s_blocksize * map.m_len;791	}792	return ret;793}794 795int ext4_get_block(struct inode *inode, sector_t iblock,796		   struct buffer_head *bh, int create)797{798	return _ext4_get_block(inode, iblock, bh,799			       create ? EXT4_GET_BLOCKS_CREATE : 0);800}801 802/*803 * Get block function used when preparing for buffered write if we require804 * creating an unwritten extent if blocks haven't been allocated.  The extent805 * will be converted to written after the IO is complete.806 */807int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,808			     struct buffer_head *bh_result, int create)809{810	int ret = 0;811 812	ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",813		   inode->i_ino, create);814	ret = _ext4_get_block(inode, iblock, bh_result,815			       EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);816 817	/*818	 * If the buffer is marked unwritten, mark it as new to make sure it is819	 * zeroed out correctly in case of partial writes. Otherwise, there is820	 * a chance of stale data getting exposed.821	 */822	if (ret == 0 && buffer_unwritten(bh_result))823		set_buffer_new(bh_result);824 825	return ret;826}827 828/* Maximum number of blocks we map for direct IO at once. */829#define DIO_MAX_BLOCKS 4096830 831/*832 * `handle' can be NULL if create is zero833 */834struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,835				ext4_lblk_t block, int map_flags)836{837	struct ext4_map_blocks map;838	struct buffer_head *bh;839	int create = map_flags & EXT4_GET_BLOCKS_CREATE;840	bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT;841	int err;842 843	ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)844		    || handle != NULL || create == 0);845	ASSERT(create == 0 || !nowait);846 847	map.m_lblk = block;848	map.m_len = 1;849	err = ext4_map_blocks(handle, inode, &map, map_flags);850 851	if (err == 0)852		return create ? ERR_PTR(-ENOSPC) : NULL;853	if (err < 0)854		return ERR_PTR(err);855 856	if (nowait)857		return sb_find_get_block(inode->i_sb, map.m_pblk);858 859	bh = sb_getblk(inode->i_sb, map.m_pblk);860	if (unlikely(!bh))861		return ERR_PTR(-ENOMEM);862	if (map.m_flags & EXT4_MAP_NEW) {863		ASSERT(create != 0);864		ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)865			    || (handle != NULL));866 867		/*868		 * Now that we do not always journal data, we should869		 * keep in mind whether this should always journal the870		 * new buffer as metadata.  For now, regular file871		 * writes use ext4_get_block instead, so it's not a872		 * problem.873		 */874		lock_buffer(bh);875		BUFFER_TRACE(bh, "call get_create_access");876		err = ext4_journal_get_create_access(handle, inode->i_sb, bh,877						     EXT4_JTR_NONE);878		if (unlikely(err)) {879			unlock_buffer(bh);880			goto errout;881		}882		if (!buffer_uptodate(bh)) {883			memset(bh->b_data, 0, inode->i_sb->s_blocksize);884			set_buffer_uptodate(bh);885		}886		unlock_buffer(bh);887		BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");888		err = ext4_handle_dirty_metadata(handle, inode, bh);889		if (unlikely(err))890			goto errout;891	} else892		BUFFER_TRACE(bh, "not a new buffer");893	return bh;894errout:895	brelse(bh);896	return ERR_PTR(err);897}898 899struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,900			       ext4_lblk_t block, int map_flags)901{902	struct buffer_head *bh;903	int ret;904 905	bh = ext4_getblk(handle, inode, block, map_flags);906	if (IS_ERR(bh))907		return bh;908	if (!bh || ext4_buffer_uptodate(bh))909		return bh;910 911	ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);912	if (ret) {913		put_bh(bh);914		return ERR_PTR(ret);915	}916	return bh;917}918 919/* Read a contiguous batch of blocks. */920int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,921		     bool wait, struct buffer_head **bhs)922{923	int i, err;924 925	for (i = 0; i < bh_count; i++) {926		bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);927		if (IS_ERR(bhs[i])) {928			err = PTR_ERR(bhs[i]);929			bh_count = i;930			goto out_brelse;931		}932	}933 934	for (i = 0; i < bh_count; i++)935		/* Note that NULL bhs[i] is valid because of holes. */936		if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))937			ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);938 939	if (!wait)940		return 0;941 942	for (i = 0; i < bh_count; i++)943		if (bhs[i])944			wait_on_buffer(bhs[i]);945 946	for (i = 0; i < bh_count; i++) {947		if (bhs[i] && !buffer_uptodate(bhs[i])) {948			err = -EIO;949			goto out_brelse;950		}951	}952	return 0;953 954out_brelse:955	for (i = 0; i < bh_count; i++) {956		brelse(bhs[i]);957		bhs[i] = NULL;958	}959	return err;960}961 962int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,963			   struct buffer_head *head,964			   unsigned from,965			   unsigned to,966			   int *partial,967			   int (*fn)(handle_t *handle, struct inode *inode,968				     struct buffer_head *bh))969{970	struct buffer_head *bh;971	unsigned block_start, block_end;972	unsigned blocksize = head->b_size;973	int err, ret = 0;974	struct buffer_head *next;975 976	for (bh = head, block_start = 0;977	     ret == 0 && (bh != head || !block_start);978	     block_start = block_end, bh = next) {979		next = bh->b_this_page;980		block_end = block_start + blocksize;981		if (block_end <= from || block_start >= to) {982			if (partial && !buffer_uptodate(bh))983				*partial = 1;984			continue;985		}986		err = (*fn)(handle, inode, bh);987		if (!ret)988			ret = err;989	}990	return ret;991}992 993/*994 * Helper for handling dirtying of journalled data. We also mark the folio as995 * dirty so that writeback code knows about this page (and inode) contains996 * dirty data. ext4_writepages() then commits appropriate transaction to997 * make data stable.998 */999static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh)1000{1001	folio_mark_dirty(bh->b_folio);1002	return ext4_handle_dirty_metadata(handle, NULL, bh);1003}1004 1005int do_journal_get_write_access(handle_t *handle, struct inode *inode,1006				struct buffer_head *bh)1007{1008	if (!buffer_mapped(bh) || buffer_freed(bh))1009		return 0;1010	BUFFER_TRACE(bh, "get write access");1011	return ext4_journal_get_write_access(handle, inode->i_sb, bh,1012					    EXT4_JTR_NONE);1013}1014 1015int ext4_block_write_begin(handle_t *handle, struct folio *folio,1016			   loff_t pos, unsigned len,1017			   get_block_t *get_block)1018{1019	unsigned from = pos & (PAGE_SIZE - 1);1020	unsigned to = from + len;1021	struct inode *inode = folio->mapping->host;1022	unsigned block_start, block_end;1023	sector_t block;1024	int err = 0;1025	unsigned blocksize = inode->i_sb->s_blocksize;1026	unsigned bbits;1027	struct buffer_head *bh, *head, *wait[2];1028	int nr_wait = 0;1029	int i;1030	bool should_journal_data = ext4_should_journal_data(inode);1031 1032	BUG_ON(!folio_test_locked(folio));1033	BUG_ON(from > PAGE_SIZE);1034	BUG_ON(to > PAGE_SIZE);1035	BUG_ON(from > to);1036 1037	head = folio_buffers(folio);1038	if (!head)1039		head = create_empty_buffers(folio, blocksize, 0);1040	bbits = ilog2(blocksize);1041	block = (sector_t)folio->index << (PAGE_SHIFT - bbits);1042 1043	for (bh = head, block_start = 0; bh != head || !block_start;1044	    block++, block_start = block_end, bh = bh->b_this_page) {1045		block_end = block_start + blocksize;1046		if (block_end <= from || block_start >= to) {1047			if (folio_test_uptodate(folio)) {1048				set_buffer_uptodate(bh);1049			}1050			continue;1051		}1052		if (buffer_new(bh))1053			clear_buffer_new(bh);1054		if (!buffer_mapped(bh)) {1055			WARN_ON(bh->b_size != blocksize);1056			err = get_block(inode, block, bh, 1);1057			if (err)1058				break;1059			if (buffer_new(bh)) {1060				/*1061				 * We may be zeroing partial buffers or all new1062				 * buffers in case of failure. Prepare JBD2 for1063				 * that.1064				 */1065				if (should_journal_data)1066					do_journal_get_write_access(handle,1067								    inode, bh);1068				if (folio_test_uptodate(folio)) {1069					/*1070					 * Unlike __block_write_begin() we leave1071					 * dirtying of new uptodate buffers to1072					 * ->write_end() time or1073					 * folio_zero_new_buffers().1074					 */1075					set_buffer_uptodate(bh);1076					continue;1077				}1078				if (block_end > to || block_start < from)1079					folio_zero_segments(folio, to,1080							    block_end,1081							    block_start, from);1082				continue;1083			}1084		}1085		if (folio_test_uptodate(folio)) {1086			set_buffer_uptodate(bh);1087			continue;1088		}1089		if (!buffer_uptodate(bh) && !buffer_delay(bh) &&1090		    !buffer_unwritten(bh) &&1091		    (block_start < from || block_end > to)) {1092			ext4_read_bh_lock(bh, 0, false);1093			wait[nr_wait++] = bh;1094		}1095	}1096	/*1097	 * If we issued read requests, let them complete.1098	 */1099	for (i = 0; i < nr_wait; i++) {1100		wait_on_buffer(wait[i]);1101		if (!buffer_uptodate(wait[i]))1102			err = -EIO;1103	}1104	if (unlikely(err)) {1105		if (should_journal_data)1106			ext4_journalled_zero_new_buffers(handle, inode, folio,1107							 from, to);1108		else1109			folio_zero_new_buffers(folio, from, to);1110	} else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {1111		for (i = 0; i < nr_wait; i++) {1112			int err2;1113 1114			err2 = fscrypt_decrypt_pagecache_blocks(folio,1115						blocksize, bh_offset(wait[i]));1116			if (err2) {1117				clear_buffer_uptodate(wait[i]);1118				err = err2;1119			}1120		}1121	}1122 1123	return err;1124}1125 1126/*1127 * To preserve ordering, it is essential that the hole instantiation and1128 * the data write be encapsulated in a single transaction.  We cannot1129 * close off a transaction and start a new one between the ext4_get_block()1130 * and the ext4_write_end().  So doing the jbd2_journal_start at the start of1131 * ext4_write_begin() is the right place.1132 */1133static int ext4_write_begin(struct file *file, struct address_space *mapping,1134			    loff_t pos, unsigned len,1135			    struct folio **foliop, void **fsdata)1136{1137	struct inode *inode = mapping->host;1138	int ret, needed_blocks;1139	handle_t *handle;1140	int retries = 0;1141	struct folio *folio;1142	pgoff_t index;1143	unsigned from, to;1144 1145	if (unlikely(ext4_forced_shutdown(inode->i_sb)))1146		return -EIO;1147 1148	trace_ext4_write_begin(inode, pos, len);1149	/*1150	 * Reserve one block more for addition to orphan list in case1151	 * we allocate blocks but write fails for some reason1152	 */1153	needed_blocks = ext4_writepage_trans_blocks(inode) + 1;1154	index = pos >> PAGE_SHIFT;1155	from = pos & (PAGE_SIZE - 1);1156	to = from + len;1157 1158	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {1159		ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,1160						    foliop);1161		if (ret < 0)1162			return ret;1163		if (ret == 1)1164			return 0;1165	}1166 1167	/*1168	 * __filemap_get_folio() can take a long time if the1169	 * system is thrashing due to memory pressure, or if the folio1170	 * is being written back.  So grab it first before we start1171	 * the transaction handle.  This also allows us to allocate1172	 * the folio (if needed) without using GFP_NOFS.1173	 */1174retry_grab:1175	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,1176					mapping_gfp_mask(mapping));1177	if (IS_ERR(folio))1178		return PTR_ERR(folio);1179	/*1180	 * The same as page allocation, we prealloc buffer heads before1181	 * starting the handle.1182	 */1183	if (!folio_buffers(folio))1184		create_empty_buffers(folio, inode->i_sb->s_blocksize, 0);1185 1186	folio_unlock(folio);1187 1188retry_journal:1189	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);1190	if (IS_ERR(handle)) {1191		folio_put(folio);1192		return PTR_ERR(handle);1193	}1194 1195	folio_lock(folio);1196	if (folio->mapping != mapping) {1197		/* The folio got truncated from under us */1198		folio_unlock(folio);1199		folio_put(folio);1200		ext4_journal_stop(handle);1201		goto retry_grab;1202	}1203	/* In case writeback began while the folio was unlocked */1204	folio_wait_stable(folio);1205 1206	if (ext4_should_dioread_nolock(inode))1207		ret = ext4_block_write_begin(handle, folio, pos, len,1208					     ext4_get_block_unwritten);1209	else1210		ret = ext4_block_write_begin(handle, folio, pos, len,1211					     ext4_get_block);1212	if (!ret && ext4_should_journal_data(inode)) {1213		ret = ext4_walk_page_buffers(handle, inode,1214					     folio_buffers(folio), from, to,1215					     NULL, do_journal_get_write_access);1216	}1217 1218	if (ret) {1219		bool extended = (pos + len > inode->i_size) &&1220				!ext4_verity_in_progress(inode);1221 1222		folio_unlock(folio);1223		/*1224		 * ext4_block_write_begin may have instantiated a few blocks1225		 * outside i_size.  Trim these off again. Don't need1226		 * i_size_read because we hold i_rwsem.1227		 *1228		 * Add inode to orphan list in case we crash before1229		 * truncate finishes1230		 */1231		if (extended && ext4_can_truncate(inode))1232			ext4_orphan_add(handle, inode);1233 1234		ext4_journal_stop(handle);1235		if (extended) {1236			ext4_truncate_failed_write(inode);1237			/*1238			 * If truncate failed early the inode might1239			 * still be on the orphan list; we need to1240			 * make sure the inode is removed from the1241			 * orphan list in that case.1242			 */1243			if (inode->i_nlink)1244				ext4_orphan_del(NULL, inode);1245		}1246 1247		if (ret == -ENOSPC &&1248		    ext4_should_retry_alloc(inode->i_sb, &retries))1249			goto retry_journal;1250		folio_put(folio);1251		return ret;1252	}1253	*foliop = folio;1254	return ret;1255}1256 1257/* For write_end() in data=journal mode */1258static int write_end_fn(handle_t *handle, struct inode *inode,1259			struct buffer_head *bh)1260{1261	int ret;1262	if (!buffer_mapped(bh) || buffer_freed(bh))1263		return 0;1264	set_buffer_uptodate(bh);1265	ret = ext4_dirty_journalled_data(handle, bh);1266	clear_buffer_meta(bh);1267	clear_buffer_prio(bh);1268	return ret;1269}1270 1271/*1272 * We need to pick up the new inode size which generic_commit_write gave us1273 * `file' can be NULL - eg, when called from page_symlink().1274 *1275 * ext4 never places buffers on inode->i_mapping->i_private_list.  metadata1276 * buffers are managed internally.1277 */1278static int ext4_write_end(struct file *file,1279			  struct address_space *mapping,1280			  loff_t pos, unsigned len, unsigned copied,1281			  struct folio *folio, void *fsdata)1282{1283	handle_t *handle = ext4_journal_current_handle();1284	struct inode *inode = mapping->host;1285	loff_t old_size = inode->i_size;1286	int ret = 0, ret2;1287	int i_size_changed = 0;1288	bool verity = ext4_verity_in_progress(inode);1289 1290	trace_ext4_write_end(inode, pos, len, copied);1291 1292	if (ext4_has_inline_data(inode) &&1293	    ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))1294		return ext4_write_inline_data_end(inode, pos, len, copied,1295						  folio);1296 1297	copied = block_write_end(file, mapping, pos, len, copied, folio, fsdata);1298	/*1299	 * it's important to update i_size while still holding folio lock:1300	 * page writeout could otherwise come in and zero beyond i_size.1301	 *1302	 * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree1303	 * blocks are being written past EOF, so skip the i_size update.1304	 */1305	if (!verity)1306		i_size_changed = ext4_update_inode_size(inode, pos + copied);1307	folio_unlock(folio);1308	folio_put(folio);1309 1310	if (old_size < pos && !verity)1311		pagecache_isize_extended(inode, old_size, pos);1312	/*1313	 * Don't mark the inode dirty under folio lock. First, it unnecessarily1314	 * makes the holding time of folio lock longer. Second, it forces lock1315	 * ordering of folio lock and transaction start for journaling1316	 * filesystems.1317	 */1318	if (i_size_changed)1319		ret = ext4_mark_inode_dirty(handle, inode);1320 1321	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))1322		/* if we have allocated more blocks and copied1323		 * less. We will have blocks allocated outside1324		 * inode->i_size. So truncate them1325		 */1326		ext4_orphan_add(handle, inode);1327 1328	ret2 = ext4_journal_stop(handle);1329	if (!ret)1330		ret = ret2;1331 1332	if (pos + len > inode->i_size && !verity) {1333		ext4_truncate_failed_write(inode);1334		/*1335		 * If truncate failed early the inode might still be1336		 * on the orphan list; we need to make sure the inode1337		 * is removed from the orphan list in that case.1338		 */1339		if (inode->i_nlink)1340			ext4_orphan_del(NULL, inode);1341	}1342 1343	return ret ? ret : copied;1344}1345 1346/*1347 * This is a private version of folio_zero_new_buffers() which doesn't1348 * set the buffer to be dirty, since in data=journalled mode we need1349 * to call ext4_dirty_journalled_data() instead.1350 */1351static void ext4_journalled_zero_new_buffers(handle_t *handle,1352					    struct inode *inode,1353					    struct folio *folio,1354					    unsigned from, unsigned to)1355{1356	unsigned int block_start = 0, block_end;1357	struct buffer_head *head, *bh;1358 1359	bh = head = folio_buffers(folio);1360	do {1361		block_end = block_start + bh->b_size;1362		if (buffer_new(bh)) {1363			if (block_end > from && block_start < to) {1364				if (!folio_test_uptodate(folio)) {1365					unsigned start, size;1366 1367					start = max(from, block_start);1368					size = min(to, block_end) - start;1369 1370					folio_zero_range(folio, start, size);1371				}1372				clear_buffer_new(bh);1373				write_end_fn(handle, inode, bh);1374			}1375		}1376		block_start = block_end;1377		bh = bh->b_this_page;1378	} while (bh != head);1379}1380 1381static int ext4_journalled_write_end(struct file *file,1382				     struct address_space *mapping,1383				     loff_t pos, unsigned len, unsigned copied,1384				     struct folio *folio, void *fsdata)1385{1386	handle_t *handle = ext4_journal_current_handle();1387	struct inode *inode = mapping->host;1388	loff_t old_size = inode->i_size;1389	int ret = 0, ret2;1390	int partial = 0;1391	unsigned from, to;1392	int size_changed = 0;1393	bool verity = ext4_verity_in_progress(inode);1394 1395	trace_ext4_journalled_write_end(inode, pos, len, copied);1396	from = pos & (PAGE_SIZE - 1);1397	to = from + len;1398 1399	BUG_ON(!ext4_handle_valid(handle));1400 1401	if (ext4_has_inline_data(inode))1402		return ext4_write_inline_data_end(inode, pos, len, copied,1403						  folio);1404 1405	if (unlikely(copied < len) && !folio_test_uptodate(folio)) {1406		copied = 0;1407		ext4_journalled_zero_new_buffers(handle, inode, folio,1408						 from, to);1409	} else {1410		if (unlikely(copied < len))1411			ext4_journalled_zero_new_buffers(handle, inode, folio,1412							 from + copied, to);1413		ret = ext4_walk_page_buffers(handle, inode,1414					     folio_buffers(folio),1415					     from, from + copied, &partial,1416					     write_end_fn);1417		if (!partial)1418			folio_mark_uptodate(folio);1419	}1420	if (!verity)1421		size_changed = ext4_update_inode_size(inode, pos + copied);1422	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;1423	folio_unlock(folio);1424	folio_put(folio);1425 1426	if (old_size < pos && !verity)1427		pagecache_isize_extended(inode, old_size, pos);1428 1429	if (size_changed) {1430		ret2 = ext4_mark_inode_dirty(handle, inode);1431		if (!ret)1432			ret = ret2;1433	}1434 1435	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))1436		/* if we have allocated more blocks and copied1437		 * less. We will have blocks allocated outside1438		 * inode->i_size. So truncate them1439		 */1440		ext4_orphan_add(handle, inode);1441 1442	ret2 = ext4_journal_stop(handle);1443	if (!ret)1444		ret = ret2;1445	if (pos + len > inode->i_size && !verity) {1446		ext4_truncate_failed_write(inode);1447		/*1448		 * If truncate failed early the inode might still be1449		 * on the orphan list; we need to make sure the inode1450		 * is removed from the orphan list in that case.1451		 */1452		if (inode->i_nlink)1453			ext4_orphan_del(NULL, inode);1454	}1455 1456	return ret ? ret : copied;1457}1458 1459/*1460 * Reserve space for 'nr_resv' clusters1461 */1462static int ext4_da_reserve_space(struct inode *inode, int nr_resv)1463{1464	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);1465	struct ext4_inode_info *ei = EXT4_I(inode);1466	int ret;1467 1468	/*1469	 * We will charge metadata quota at writeout time; this saves1470	 * us from metadata over-estimation, though we may go over by1471	 * a small amount in the end.  Here we just reserve for data.1472	 */1473	ret = dquot_reserve_block(inode, EXT4_C2B(sbi, nr_resv));1474	if (ret)1475		return ret;1476 1477	spin_lock(&ei->i_block_reservation_lock);1478	if (ext4_claim_free_clusters(sbi, nr_resv, 0)) {1479		spin_unlock(&ei->i_block_reservation_lock);1480		dquot_release_reservation_block(inode, EXT4_C2B(sbi, nr_resv));1481		return -ENOSPC;1482	}1483	ei->i_reserved_data_blocks += nr_resv;1484	trace_ext4_da_reserve_space(inode, nr_resv);1485	spin_unlock(&ei->i_block_reservation_lock);1486 1487	return 0;       /* success */1488}1489 1490void ext4_da_release_space(struct inode *inode, int to_free)1491{1492	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);1493	struct ext4_inode_info *ei = EXT4_I(inode);1494 1495	if (!to_free)1496		return;		/* Nothing to release, exit */1497 1498	spin_lock(&EXT4_I(inode)->i_block_reservation_lock);1499 1500	trace_ext4_da_release_space(inode, to_free);1501	if (unlikely(to_free > ei->i_reserved_data_blocks)) {1502		/*1503		 * if there aren't enough reserved blocks, then the1504		 * counter is messed up somewhere.  Since this1505		 * function is called from invalidate page, it's1506		 * harmless to return without any action.1507		 */1508		ext4_warning(inode->i_sb, "ext4_da_release_space: "1509			 "ino %lu, to_free %d with only %d reserved "1510			 "data blocks", inode->i_ino, to_free,1511			 ei->i_reserved_data_blocks);1512		WARN_ON(1);1513		to_free = ei->i_reserved_data_blocks;1514	}1515	ei->i_reserved_data_blocks -= to_free;1516 1517	/* update fs dirty data blocks counter */1518	percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);1519 1520	spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);1521 1522	dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));1523}1524 1525/*1526 * Delayed allocation stuff1527 */1528 1529struct mpage_da_data {1530	/* These are input fields for ext4_do_writepages() */1531	struct inode *inode;1532	struct writeback_control *wbc;1533	unsigned int can_map:1;	/* Can writepages call map blocks? */1534 1535	/* These are internal state of ext4_do_writepages() */1536	pgoff_t first_page;	/* The first page to write */1537	pgoff_t next_page;	/* Current page to examine */1538	pgoff_t last_page;	/* Last page to examine */1539	/*1540	 * Extent to map - this can be after first_page because that can be1541	 * fully mapped. We somewhat abuse m_flags to store whether the extent1542	 * is delalloc or unwritten.1543	 */1544	struct ext4_map_blocks map;1545	struct ext4_io_submit io_submit;	/* IO submission data */1546	unsigned int do_map:1;1547	unsigned int scanned_until_end:1;1548	unsigned int journalled_more_data:1;1549};1550 1551static void mpage_release_unused_pages(struct mpage_da_data *mpd,1552				       bool invalidate)1553{1554	unsigned nr, i;1555	pgoff_t index, end;1556	struct folio_batch fbatch;1557	struct inode *inode = mpd->inode;1558	struct address_space *mapping = inode->i_mapping;1559 1560	/* This is necessary when next_page == 0. */1561	if (mpd->first_page >= mpd->next_page)1562		return;1563 1564	mpd->scanned_until_end = 0;1565	index = mpd->first_page;1566	end   = mpd->next_page - 1;1567	if (invalidate) {1568		ext4_lblk_t start, last;1569		start = index << (PAGE_SHIFT - inode->i_blkbits);1570		last = end << (PAGE_SHIFT - inode->i_blkbits);1571 1572		/*1573		 * avoid racing with extent status tree scans made by1574		 * ext4_insert_delayed_block()1575		 */1576		down_write(&EXT4_I(inode)->i_data_sem);1577		ext4_es_remove_extent(inode, start, last - start + 1);1578		up_write(&EXT4_I(inode)->i_data_sem);1579	}1580 1581	folio_batch_init(&fbatch);1582	while (index <= end) {1583		nr = filemap_get_folios(mapping, &index, end, &fbatch);1584		if (nr == 0)1585			break;1586		for (i = 0; i < nr; i++) {1587			struct folio *folio = fbatch.folios[i];1588 1589			if (folio->index < mpd->first_page)1590				continue;1591			if (folio_next_index(folio) - 1 > end)1592				continue;1593			BUG_ON(!folio_test_locked(folio));1594			BUG_ON(folio_test_writeback(folio));1595			if (invalidate) {1596				if (folio_mapped(folio))1597					folio_clear_dirty_for_io(folio);1598				block_invalidate_folio(folio, 0,1599						folio_size(folio));1600				folio_clear_uptodate(folio);1601			}1602			folio_unlock(folio);1603		}1604		folio_batch_release(&fbatch);1605	}1606}1607 1608static void ext4_print_free_blocks(struct inode *inode)1609{1610	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);1611	struct super_block *sb = inode->i_sb;1612	struct ext4_inode_info *ei = EXT4_I(inode);1613 1614	ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",1615	       EXT4_C2B(EXT4_SB(inode->i_sb),1616			ext4_count_free_clusters(sb)));1617	ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");1618	ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",1619	       (long long) EXT4_C2B(EXT4_SB(sb),1620		percpu_counter_sum(&sbi->s_freeclusters_counter)));1621	ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",1622	       (long long) EXT4_C2B(EXT4_SB(sb),1623		percpu_counter_sum(&sbi->s_dirtyclusters_counter)));1624	ext4_msg(sb, KERN_CRIT, "Block reservation details");1625	ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",1626		 ei->i_reserved_data_blocks);1627	return;1628}1629 1630/*1631 * Check whether the cluster containing lblk has been allocated or has1632 * delalloc reservation.1633 *1634 * Returns 0 if the cluster doesn't have either, 1 if it has delalloc1635 * reservation, 2 if it's already been allocated, negative error code on1636 * failure.1637 */1638static int ext4_clu_alloc_state(struct inode *inode, ext4_lblk_t lblk)1639{1640	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);1641	int ret;1642 1643	/* Has delalloc reservation? */1644	if (ext4_es_scan_clu(inode, &ext4_es_is_delayed, lblk))1645		return 1;1646 1647	/* Already been allocated? */1648	if (ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk))1649		return 2;1650	ret = ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk));1651	if (ret < 0)1652		return ret;1653	if (ret > 0)1654		return 2;1655 1656	return 0;1657}1658 1659/*1660 * ext4_insert_delayed_blocks - adds a multiple delayed blocks to the extents1661 *                              status tree, incrementing the reserved1662 *                              cluster/block count or making pending1663 *                              reservations where needed1664 *1665 * @inode - file containing the newly added block1666 * @lblk - start logical block to be added1667 * @len - length of blocks to be added1668 *1669 * Returns 0 on success, negative error code on failure.1670 */1671static int ext4_insert_delayed_blocks(struct inode *inode, ext4_lblk_t lblk,1672				      ext4_lblk_t len)1673{1674	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);1675	int ret;1676	bool lclu_allocated = false;1677	bool end_allocated = false;1678	ext4_lblk_t resv_clu;1679	ext4_lblk_t end = lblk + len - 1;1680 1681	/*1682	 * If the cluster containing lblk or end is shared with a delayed,1683	 * written, or unwritten extent in a bigalloc file system, it's1684	 * already been accounted for and does not need to be reserved.1685	 * A pending reservation must be made for the cluster if it's1686	 * shared with a written or unwritten extent and doesn't already1687	 * have one.  Written and unwritten extents can be purged from the1688	 * extents status tree if the system is under memory pressure, so1689	 * it's necessary to examine the extent tree if a search of the1690	 * extents status tree doesn't get a match.1691	 */1692	if (sbi->s_cluster_ratio == 1) {1693		ret = ext4_da_reserve_space(inode, len);1694		if (ret != 0)   /* ENOSPC */1695			return ret;1696	} else {   /* bigalloc */1697		resv_clu = EXT4_B2C(sbi, end) - EXT4_B2C(sbi, lblk) + 1;1698 1699		ret = ext4_clu_alloc_state(inode, lblk);1700		if (ret < 0)1701			return ret;1702		if (ret > 0) {1703			resv_clu--;1704			lclu_allocated = (ret == 2);1705		}1706 1707		if (EXT4_B2C(sbi, lblk) != EXT4_B2C(sbi, end)) {1708			ret = ext4_clu_alloc_state(inode, end);1709			if (ret < 0)1710				return ret;1711			if (ret > 0) {1712				resv_clu--;1713				end_allocated = (ret == 2);1714			}1715		}1716 1717		if (resv_clu) {1718			ret = ext4_da_reserve_space(inode, resv_clu);1719			if (ret != 0)   /* ENOSPC */1720				return ret;1721		}1722	}1723 1724	ext4_es_insert_delayed_extent(inode, lblk, len, lclu_allocated,1725				      end_allocated);1726	return 0;1727}1728 1729/*1730 * Looks up the requested blocks and sets the delalloc extent map.1731 * First try to look up for the extent entry that contains the requested1732 * blocks in the extent status tree without i_data_sem, then try to look1733 * up for the ondisk extent mapping with i_data_sem in read mode,1734 * finally hold i_data_sem in write mode, looks up again and add a1735 * delalloc extent entry if it still couldn't find any extent. Pass out1736 * the mapped extent through @map and return 0 on success.1737 */1738static int ext4_da_map_blocks(struct inode *inode, struct ext4_map_blocks *map)1739{1740	struct extent_status es;1741	int retval;1742#ifdef ES_AGGRESSIVE_TEST1743	struct ext4_map_blocks orig_map;1744 1745	memcpy(&orig_map, map, sizeof(*map));1746#endif1747 1748	map->m_flags = 0;1749	ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,1750		  (unsigned long) map->m_lblk);1751 1752	/* Lookup extent status tree firstly */1753	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {1754		map->m_len = min_t(unsigned int, map->m_len,1755				   es.es_len - (map->m_lblk - es.es_lblk));1756 1757		if (ext4_es_is_hole(&es))1758			goto add_delayed;1759 1760found:1761		/*1762		 * Delayed extent could be allocated by fallocate.1763		 * So we need to check it.1764		 */1765		if (ext4_es_is_delayed(&es)) {1766			map->m_flags |= EXT4_MAP_DELAYED;1767			return 0;1768		}1769 1770		map->m_pblk = ext4_es_pblock(&es) + map->m_lblk - es.es_lblk;1771		if (ext4_es_is_written(&es))1772			map->m_flags |= EXT4_MAP_MAPPED;1773		else if (ext4_es_is_unwritten(&es))1774			map->m_flags |= EXT4_MAP_UNWRITTEN;1775		else1776			BUG();1777 1778#ifdef ES_AGGRESSIVE_TEST1779		ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);1780#endif1781		return 0;1782	}1783 1784	/*1785	 * Try to see if we can get the block without requesting a new1786	 * file system block.1787	 */1788	down_read(&EXT4_I(inode)->i_data_sem);1789	if (ext4_has_inline_data(inode))1790		retval = 0;1791	else1792		retval = ext4_map_query_blocks(NULL, inode, map);1793	up_read(&EXT4_I(inode)->i_data_sem);1794	if (retval)1795		return retval < 0 ? retval : 0;1796 1797add_delayed:1798	down_write(&EXT4_I(inode)->i_data_sem);1799	/*1800	 * Page fault path (ext4_page_mkwrite does not take i_rwsem)1801	 * and fallocate path (no folio lock) can race. Make sure we1802	 * lookup the extent status tree here again while i_data_sem1803	 * is held in write mode, before inserting a new da entry in1804	 * the extent status tree.1805	 */1806	if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {1807		map->m_len = min_t(unsigned int, map->m_len,1808				   es.es_len - (map->m_lblk - es.es_lblk));1809 1810		if (!ext4_es_is_hole(&es)) {1811			up_write(&EXT4_I(inode)->i_data_sem);1812			goto found;1813		}1814	} else if (!ext4_has_inline_data(inode)) {1815		retval = ext4_map_query_blocks(NULL, inode, map);1816		if (retval) {1817			up_write(&EXT4_I(inode)->i_data_sem);1818			return retval < 0 ? retval : 0;1819		}1820	}1821 1822	map->m_flags |= EXT4_MAP_DELAYED;1823	retval = ext4_insert_delayed_blocks(inode, map->m_lblk, map->m_len);1824	up_write(&EXT4_I(inode)->i_data_sem);1825 1826	return retval;1827}1828 1829/*1830 * This is a special get_block_t callback which is used by1831 * ext4_da_write_begin().  It will either return mapped block or1832 * reserve space for a single block.1833 *1834 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.1835 * We also have b_blocknr = -1 and b_bdev initialized properly1836 *1837 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.1838 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev1839 * initialized properly.1840 */1841int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,1842			   struct buffer_head *bh, int create)1843{1844	struct ext4_map_blocks map;1845	sector_t invalid_block = ~((sector_t) 0xffff);1846	int ret = 0;1847 1848	BUG_ON(create == 0);1849	BUG_ON(bh->b_size != inode->i_sb->s_blocksize);1850 1851	if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))1852		invalid_block = ~0;1853 1854	map.m_lblk = iblock;1855	map.m_len = 1;1856 1857	/*1858	 * first, we need to know whether the block is allocated already1859	 * preallocated blocks are unmapped but should treated1860	 * the same as allocated blocks.1861	 */1862	ret = ext4_da_map_blocks(inode, &map);1863	if (ret < 0)1864		return ret;1865 1866	if (map.m_flags & EXT4_MAP_DELAYED) {1867		map_bh(bh, inode->i_sb, invalid_block);1868		set_buffer_new(bh);1869		set_buffer_delay(bh);1870		return 0;1871	}1872 1873	map_bh(bh, inode->i_sb, map.m_pblk);1874	ext4_update_bh_state(bh, map.m_flags);1875 1876	if (buffer_unwritten(bh)) {1877		/* A delayed write to unwritten bh should be marked1878		 * new and mapped.  Mapped ensures that we don't do1879		 * get_block multiple times when we write to the same1880		 * offset and new ensures that we do proper zero out1881		 * for partial write.1882		 */1883		set_buffer_new(bh);1884		set_buffer_mapped(bh);1885	}1886	return 0;1887}1888 1889static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio)1890{1891	mpd->first_page += folio_nr_pages(folio);1892	folio_unlock(folio);1893}1894 1895static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio)1896{1897	size_t len;1898	loff_t size;1899	int err;1900 1901	BUG_ON(folio->index != mpd->first_page);1902	folio_clear_dirty_for_io(folio);1903	/*1904	 * We have to be very careful here!  Nothing protects writeback path1905	 * against i_size changes and the page can be writeably mapped into1906	 * page tables. So an application can be growing i_size and writing1907	 * data through mmap while writeback runs. folio_clear_dirty_for_io()1908	 * write-protects our page in page tables and the page cannot get1909	 * written to again until we release folio lock. So only after1910	 * folio_clear_dirty_for_io() we are safe to sample i_size for1911	 * ext4_bio_write_folio() to zero-out tail of the written page. We rely1912	 * on the barrier provided by folio_test_clear_dirty() in1913	 * folio_clear_dirty_for_io() to make sure i_size is really sampled only1914	 * after page tables are updated.1915	 */1916	size = i_size_read(mpd->inode);1917	len = folio_size(folio);1918	if (folio_pos(folio) + len > size &&1919	    !ext4_verity_in_progress(mpd->inode))1920		len = size & (len - 1);1921	err = ext4_bio_write_folio(&mpd->io_submit, folio, len);1922	if (!err)1923		mpd->wbc->nr_to_write--;1924 1925	return err;1926}1927 1928#define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))1929 1930/*1931 * mballoc gives us at most this number of blocks...1932 * XXX: That seems to be only a limitation of ext4_mb_normalize_request().1933 * The rest of mballoc seems to handle chunks up to full group size.1934 */1935#define MAX_WRITEPAGES_EXTENT_LEN 20481936 1937/*1938 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map1939 *1940 * @mpd - extent of blocks1941 * @lblk - logical number of the block in the file1942 * @bh - buffer head we want to add to the extent1943 *1944 * The function is used to collect contig. blocks in the same state. If the1945 * buffer doesn't require mapping for writeback and we haven't started the1946 * extent of buffers to map yet, the function returns 'true' immediately - the1947 * caller can write the buffer right away. Otherwise the function returns true1948 * if the block has been added to the extent, false if the block couldn't be1949 * added.1950 */1951static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,1952				   struct buffer_head *bh)1953{1954	struct ext4_map_blocks *map = &mpd->map;1955 1956	/* Buffer that doesn't need mapping for writeback? */1957	if (!buffer_dirty(bh) || !buffer_mapped(bh) ||1958	    (!buffer_delay(bh) && !buffer_unwritten(bh))) {1959		/* So far no extent to map => we write the buffer right away */1960		if (map->m_len == 0)1961			return true;1962		return false;1963	}1964 1965	/* First block in the extent? */1966	if (map->m_len == 0) {1967		/* We cannot map unless handle is started... */1968		if (!mpd->do_map)1969			return false;1970		map->m_lblk = lblk;1971		map->m_len = 1;1972		map->m_flags = bh->b_state & BH_FLAGS;1973		return true;1974	}1975 1976	/* Don't go larger than mballoc is willing to allocate */1977	if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)1978		return false;1979 1980	/* Can we merge the block to our big extent? */1981	if (lblk == map->m_lblk + map->m_len &&1982	    (bh->b_state & BH_FLAGS) == map->m_flags) {1983		map->m_len++;1984		return true;1985	}1986	return false;1987}1988 1989/*1990 * mpage_process_page_bufs - submit page buffers for IO or add them to extent1991 *1992 * @mpd - extent of blocks for mapping1993 * @head - the first buffer in the page1994 * @bh - buffer we should start processing from1995 * @lblk - logical number of the block in the file corresponding to @bh1996 *1997 * Walk through page buffers from @bh upto @head (exclusive) and either submit1998 * the page for IO if all buffers in this page were mapped and there's no1999 * accumulated extent of buffers to map or add buffers in the page to the2000 * extent of buffers to map. The function returns 1 if the caller can continue2001 * by processing the next page, 0 if it should stop adding buffers to the2002 * extent to map because we cannot extend it anymore. It can also return value2003 * < 0 in case of error during IO submission.2004 */2005static int mpage_process_page_bufs(struct mpage_da_data *mpd,2006				   struct buffer_head *head,2007				   struct buffer_head *bh,2008				   ext4_lblk_t lblk)2009{2010	struct inode *inode = mpd->inode;2011	int err;2012	ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)2013							>> inode->i_blkbits;2014 2015	if (ext4_verity_in_progress(inode))2016		blocks = EXT_MAX_BLOCKS;2017 2018	do {2019		BUG_ON(buffer_locked(bh));2020 2021		if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {2022			/* Found extent to map? */2023			if (mpd->map.m_len)2024				return 0;2025			/* Buffer needs mapping and handle is not started? */2026			if (!mpd->do_map)2027				return 0;2028			/* Everything mapped so far and we hit EOF */2029			break;2030		}2031	} while (lblk++, (bh = bh->b_this_page) != head);2032	/* So far everything mapped? Submit the page for IO. */2033	if (mpd->map.m_len == 0) {2034		err = mpage_submit_folio(mpd, head->b_folio);2035		if (err < 0)2036			return err;2037		mpage_folio_done(mpd, head->b_folio);2038	}2039	if (lblk >= blocks) {2040		mpd->scanned_until_end = 1;2041		return 0;2042	}2043	return 1;2044}2045 2046/*2047 * mpage_process_folio - update folio buffers corresponding to changed extent2048 *			 and may submit fully mapped page for IO2049 * @mpd: description of extent to map, on return next extent to map2050 * @folio: Contains these buffers.2051 * @m_lblk: logical block mapping.2052 * @m_pblk: corresponding physical mapping.2053 * @map_bh: determines on return whether this page requires any further2054 *		  mapping or not.2055 *2056 * Scan given folio buffers corresponding to changed extent and update buffer2057 * state according to new extent state.2058 * We map delalloc buffers to their physical location, clear unwritten bits.2059 * If the given folio is not fully mapped, we update @mpd to the next extent in2060 * the given folio that needs mapping & return @map_bh as true.2061 */2062static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio,2063			      ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,2064			      bool *map_bh)2065{2066	struct buffer_head *head, *bh;2067	ext4_io_end_t *io_end = mpd->io_submit.io_end;2068	ext4_lblk_t lblk = *m_lblk;2069	ext4_fsblk_t pblock = *m_pblk;2070	int err = 0;2071	int blkbits = mpd->inode->i_blkbits;2072	ssize_t io_end_size = 0;2073	struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);2074 2075	bh = head = folio_buffers(folio);2076	do {2077		if (lblk < mpd->map.m_lblk)2078			continue;2079		if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {2080			/*2081			 * Buffer after end of mapped extent.2082			 * Find next buffer in the folio to map.2083			 */2084			mpd->map.m_len = 0;2085			mpd->map.m_flags = 0;2086			io_end_vec->size += io_end_size;2087 2088			err = mpage_process_page_bufs(mpd, head, bh, lblk);2089			if (err > 0)2090				err = 0;2091			if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {2092				io_end_vec = ext4_alloc_io_end_vec(io_end);2093				if (IS_ERR(io_end_vec)) {2094					err = PTR_ERR(io_end_vec);2095					goto out;2096				}2097				io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;2098			}2099			*map_bh = true;2100			goto out;2101		}2102		if (buffer_delay(bh)) {2103			clear_buffer_delay(bh);2104			bh->b_blocknr = pblock++;2105		}2106		clear_buffer_unwritten(bh);2107		io_end_size += (1 << blkbits);2108	} while (lblk++, (bh = bh->b_this_page) != head);2109 2110	io_end_vec->size += io_end_size;2111	*map_bh = false;2112out:2113	*m_lblk = lblk;2114	*m_pblk = pblock;2115	return err;2116}2117 2118/*2119 * mpage_map_buffers - update buffers corresponding to changed extent and2120 *		       submit fully mapped pages for IO2121 *2122 * @mpd - description of extent to map, on return next extent to map2123 *2124 * Scan buffers corresponding to changed extent (we expect corresponding pages2125 * to be already locked) and update buffer state according to new extent state.2126 * We map delalloc buffers to their physical location, clear unwritten bits,2127 * and mark buffers as uninit when we perform writes to unwritten extents2128 * and do extent conversion after IO is finished. If the last page is not fully2129 * mapped, we update @map to the next extent in the last page that needs2130 * mapping. Otherwise we submit the page for IO.2131 */2132static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)2133{2134	struct folio_batch fbatch;2135	unsigned nr, i;2136	struct inode *inode = mpd->inode;2137	int bpp_bits = PAGE_SHIFT - inode->i_blkbits;2138	pgoff_t start, end;2139	ext4_lblk_t lblk;2140	ext4_fsblk_t pblock;2141	int err;2142	bool map_bh = false;2143 2144	start = mpd->map.m_lblk >> bpp_bits;2145	end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;2146	lblk = start << bpp_bits;2147	pblock = mpd->map.m_pblk;2148 2149	folio_batch_init(&fbatch);2150	while (start <= end) {2151		nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch);2152		if (nr == 0)2153			break;2154		for (i = 0; i < nr; i++) {2155			struct folio *folio = fbatch.folios[i];2156 2157			err = mpage_process_folio(mpd, folio, &lblk, &pblock,2158						 &map_bh);2159			/*2160			 * If map_bh is true, means page may require further bh2161			 * mapping, or maybe the page was submitted for IO.2162			 * So we return to call further extent mapping.2163			 */2164			if (err < 0 || map_bh)2165				goto out;2166			/* Page fully mapped - let IO run! */2167			err = mpage_submit_folio(mpd, folio);2168			if (err < 0)2169				goto out;2170			mpage_folio_done(mpd, folio);2171		}2172		folio_batch_release(&fbatch);2173	}2174	/* Extent fully mapped and matches with page boundary. We are done. */2175	mpd->map.m_len = 0;2176	mpd->map.m_flags = 0;2177	return 0;2178out:2179	folio_batch_release(&fbatch);2180	return err;2181}2182 2183static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)2184{2185	struct inode *inode = mpd->inode;2186	struct ext4_map_blocks *map = &mpd->map;2187	int get_blocks_flags;2188	int err, dioread_nolock;2189 2190	trace_ext4_da_write_pages_extent(inode, map);2191	/*2192	 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or2193	 * to convert an unwritten extent to be initialized (in the case2194	 * where we have written into one or more preallocated blocks).  It is2195	 * possible that we're going to need more metadata blocks than2196	 * previously reserved. However we must not fail because we're in2197	 * writeback and there is nothing we can do about it so it might result2198	 * in data loss.  So use reserved blocks to allocate metadata if2199	 * possible.2200	 */2201	get_blocks_flags = EXT4_GET_BLOCKS_CREATE |2202			   EXT4_GET_BLOCKS_METADATA_NOFAIL |2203			   EXT4_GET_BLOCKS_IO_SUBMIT;2204	dioread_nolock = ext4_should_dioread_nolock(inode);2205	if (dioread_nolock)2206		get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;2207 2208	err = ext4_map_blocks(handle, inode, map, get_blocks_flags);2209	if (err < 0)2210		return err;2211	if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {2212		if (!mpd->io_submit.io_end->handle &&2213		    ext4_handle_valid(handle)) {2214			mpd->io_submit.io_end->handle = handle->h_rsv_handle;2215			handle->h_rsv_handle = NULL;2216		}2217		ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);2218	}2219 2220	BUG_ON(map->m_len == 0);2221	return 0;2222}2223 2224/*2225 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length2226 *				 mpd->len and submit pages underlying it for IO2227 *2228 * @handle - handle for journal operations2229 * @mpd - extent to map2230 * @give_up_on_write - we set this to true iff there is a fatal error and there2231 *                     is no hope of writing the data. The caller should discard2232 *                     dirty pages to avoid infinite loops.2233 *2234 * The function maps extent starting at mpd->lblk of length mpd->len. If it is2235 * delayed, blocks are allocated, if it is unwritten, we may need to convert2236 * them to initialized or split the described range from larger unwritten2237 * extent. Note that we need not map all the described range since allocation2238 * can return less blocks or the range is covered by more unwritten extents. We2239 * cannot map more because we are limited by reserved transaction credits. On2240 * the other hand we always make sure that the last touched page is fully2241 * mapped so that it can be written out (and thus forward progress is2242 * guaranteed). After mapping we submit all mapped pages for IO.2243 */2244static int mpage_map_and_submit_extent(handle_t *handle,2245				       struct mpage_da_data *mpd,2246				       bool *give_up_on_write)2247{2248	struct inode *inode = mpd->inode;2249	struct ext4_map_blocks *map = &mpd->map;2250	int err;2251	loff_t disksize;2252	int progress = 0;2253	ext4_io_end_t *io_end = mpd->io_submit.io_end;2254	struct ext4_io_end_vec *io_end_vec;2255 2256	io_end_vec = ext4_alloc_io_end_vec(io_end);2257	if (IS_ERR(io_end_vec))2258		return PTR_ERR(io_end_vec);2259	io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;2260	do {2261		err = mpage_map_one_extent(handle, mpd);2262		if (err < 0) {2263			struct super_block *sb = inode->i_sb;2264 2265			if (ext4_forced_shutdown(sb))2266				goto invalidate_dirty_pages;2267			/*2268			 * Let the uper layers retry transient errors.2269			 * In the case of ENOSPC, if ext4_count_free_blocks()2270			 * is non-zero, a commit should free up blocks.2271			 */2272			if ((err == -ENOMEM) ||2273			    (err == -ENOSPC && ext4_count_free_clusters(sb))) {2274				if (progress)2275					goto update_disksize;2276				return err;2277			}2278			ext4_msg(sb, KERN_CRIT,2279				 "Delayed block allocation failed for "2280				 "inode %lu at logical offset %llu with"2281				 " max blocks %u with error %d",2282				 inode->i_ino,2283				 (unsigned long long)map->m_lblk,2284				 (unsigned)map->m_len, -err);2285			ext4_msg(sb, KERN_CRIT,2286				 "This should not happen!! Data will "2287				 "be lost\n");2288			if (err == -ENOSPC)2289				ext4_print_free_blocks(inode);2290		invalidate_dirty_pages:2291			*give_up_on_write = true;2292			return err;2293		}2294		progress = 1;2295		/*2296		 * Update buffer state, submit mapped pages, and get us new2297		 * extent to map2298		 */2299		err = mpage_map_and_submit_buffers(mpd);2300		if (err < 0)2301			goto update_disksize;2302	} while (map->m_len);2303 2304update_disksize:2305	/*2306	 * Update on-disk size after IO is submitted.  Races with2307	 * truncate are avoided by checking i_size under i_data_sem.2308	 */2309	disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;2310	if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {2311		int err2;2312		loff_t i_size;2313 2314		down_write(&EXT4_I(inode)->i_data_sem);2315		i_size = i_size_read(inode);2316		if (disksize > i_size)2317			disksize = i_size;2318		if (disksize > EXT4_I(inode)->i_disksize)2319			EXT4_I(inode)->i_disksize = disksize;2320		up_write(&EXT4_I(inode)->i_data_sem);2321		err2 = ext4_mark_inode_dirty(handle, inode);2322		if (err2) {2323			ext4_error_err(inode->i_sb, -err2,2324				       "Failed to mark inode %lu dirty",2325				       inode->i_ino);2326		}2327		if (!err)2328			err = err2;2329	}2330	return err;2331}2332 2333/*2334 * Calculate the total number of credits to reserve for one writepages2335 * iteration. This is called from ext4_writepages(). We map an extent of2336 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping2337 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +2338 * bpp - 1 blocks in bpp different extents.2339 */2340static int ext4_da_writepages_trans_blocks(struct inode *inode)2341{2342	int bpp = ext4_journal_blocks_per_page(inode);2343 2344	return ext4_meta_trans_blocks(inode,2345				MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);2346}2347 2348static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio,2349				     size_t len)2350{2351	struct buffer_head *page_bufs = folio_buffers(folio);2352	struct inode *inode = folio->mapping->host;2353	int ret, err;2354 2355	ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,2356				     NULL, do_journal_get_write_access);2357	err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,2358				     NULL, write_end_fn);2359	if (ret == 0)2360		ret = err;2361	err = ext4_jbd2_inode_add_write(handle, inode, folio_pos(folio), len);2362	if (ret == 0)2363		ret = err;2364	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;2365 2366	return ret;2367}2368 2369static int mpage_journal_page_buffers(handle_t *handle,2370				      struct mpage_da_data *mpd,2371				      struct folio *folio)2372{2373	struct inode *inode = mpd->inode;2374	loff_t size = i_size_read(inode);2375	size_t len = folio_size(folio);2376 2377	folio_clear_checked(folio);2378	mpd->wbc->nr_to_write--;2379 2380	if (folio_pos(folio) + len > size &&2381	    !ext4_verity_in_progress(inode))2382		len = size & (len - 1);2383 2384	return ext4_journal_folio_buffers(handle, folio, len);2385}2386 2387/*2388 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages2389 * 				 needing mapping, submit mapped pages2390 *2391 * @mpd - where to look for pages2392 *2393 * Walk dirty pages in the mapping. If they are fully mapped, submit them for2394 * IO immediately. If we cannot map blocks, we submit just already mapped2395 * buffers in the page for IO and keep page dirty. When we can map blocks and2396 * we find a page which isn't mapped we start accumulating extent of buffers2397 * underlying these pages that needs mapping (formed by either delayed or2398 * unwritten buffers). We also lock the pages containing these buffers. The2399 * extent found is returned in @mpd structure (starting at mpd->lblk with2400 * length mpd->len blocks).2401 *2402 * Note that this function can attach bios to one io_end structure which are2403 * neither logically nor physically contiguous. Although it may seem as an2404 * unnecessary complication, it is actually inevitable in blocksize < pagesize2405 * case as we need to track IO to all buffers underlying a page in one io_end.2406 */2407static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)2408{2409	struct address_space *mapping = mpd->inode->i_mapping;2410	struct folio_batch fbatch;2411	unsigned int nr_folios;2412	pgoff_t index = mpd->first_page;2413	pgoff_t end = mpd->last_page;2414	xa_mark_t tag;2415	int i, err = 0;2416	int blkbits = mpd->inode->i_blkbits;2417	ext4_lblk_t lblk;2418	struct buffer_head *head;2419	handle_t *handle = NULL;2420	int bpp = ext4_journal_blocks_per_page(mpd->inode);2421 2422	if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)2423		tag = PAGECACHE_TAG_TOWRITE;2424	else2425		tag = PAGECACHE_TAG_DIRTY;2426 2427	mpd->map.m_len = 0;2428	mpd->next_page = index;2429	if (ext4_should_journal_data(mpd->inode)) {2430		handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE,2431					    bpp);2432		if (IS_ERR(handle))2433			return PTR_ERR(handle);2434	}2435	folio_batch_init(&fbatch);2436	while (index <= end) {2437		nr_folios = filemap_get_folios_tag(mapping, &index, end,2438				tag, &fbatch);2439		if (nr_folios == 0)2440			break;2441 2442		for (i = 0; i < nr_folios; i++) {2443			struct folio *folio = fbatch.folios[i];2444 2445			/*2446			 * Accumulated enough dirty pages? This doesn't apply2447			 * to WB_SYNC_ALL mode. For integrity sync we have to2448			 * keep going because someone may be concurrently2449			 * dirtying pages, and we might have synced a lot of2450			 * newly appeared dirty pages, but have not synced all2451			 * of the old dirty pages.2452			 */2453			if (mpd->wbc->sync_mode == WB_SYNC_NONE &&2454			    mpd->wbc->nr_to_write <=2455			    mpd->map.m_len >> (PAGE_SHIFT - blkbits))2456				goto out;2457 2458			/* If we can't merge this page, we are done. */2459			if (mpd->map.m_len > 0 && mpd->next_page != folio->index)2460				goto out;2461 2462			if (handle) {2463				err = ext4_journal_ensure_credits(handle, bpp,2464								  0);2465				if (err < 0)2466					goto out;2467			}2468 2469			folio_lock(folio);2470			/*2471			 * If the page is no longer dirty, or its mapping no2472			 * longer corresponds to inode we are writing (which2473			 * means it has been truncated or invalidated), or the2474			 * page is already under writeback and we are not doing2475			 * a data integrity writeback, skip the page2476			 */2477			if (!folio_test_dirty(folio) ||2478			    (folio_test_writeback(folio) &&2479			     (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||2480			    unlikely(folio->mapping != mapping)) {2481				folio_unlock(folio);2482				continue;2483			}2484 2485			folio_wait_writeback(folio);2486			BUG_ON(folio_test_writeback(folio));2487 2488			/*2489			 * Should never happen but for buggy code in2490			 * other subsystems that call2491			 * set_page_dirty() without properly warning2492			 * the file system first.  See [1] for more2493			 * information.2494			 *2495			 * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz2496			 */2497			if (!folio_buffers(folio)) {2498				ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index);2499				folio_clear_dirty(folio);2500				folio_unlock(folio);2501				continue;2502			}2503 2504			if (mpd->map.m_len == 0)2505				mpd->first_page = folio->index;2506			mpd->next_page = folio_next_index(folio);2507			/*2508			 * Writeout when we cannot modify metadata is simple.2509			 * Just submit the page. For data=journal mode we2510			 * first handle writeout of the page for checkpoint and2511			 * only after that handle delayed page dirtying. This2512			 * makes sure current data is checkpointed to the final2513			 * location before possibly journalling it again which2514			 * is desirable when the page is frequently dirtied2515			 * through a pin.2516			 */2517			if (!mpd->can_map) {2518				err = mpage_submit_folio(mpd, folio);2519				if (err < 0)2520					goto out;2521				/* Pending dirtying of journalled data? */2522				if (folio_test_checked(folio)) {2523					err = mpage_journal_page_buffers(handle,2524						mpd, folio);2525					if (err < 0)2526						goto out;2527					mpd->journalled_more_data = 1;2528				}2529				mpage_folio_done(mpd, folio);2530			} else {2531				/* Add all dirty buffers to mpd */2532				lblk = ((ext4_lblk_t)folio->index) <<2533					(PAGE_SHIFT - blkbits);2534				head = folio_buffers(folio);2535				err = mpage_process_page_bufs(mpd, head, head,2536						lblk);2537				if (err <= 0)2538					goto out;2539				err = 0;2540			}2541		}2542		folio_batch_release(&fbatch);2543		cond_resched();2544	}2545	mpd->scanned_until_end = 1;2546	if (handle)2547		ext4_journal_stop(handle);2548	return 0;2549out:2550	folio_batch_release(&fbatch);2551	if (handle)2552		ext4_journal_stop(handle);2553	return err;2554}2555 2556static int ext4_do_writepages(struct mpage_da_data *mpd)2557{2558	struct writeback_control *wbc = mpd->wbc;2559	pgoff_t	writeback_index = 0;2560	long nr_to_write = wbc->nr_to_write;2561	int range_whole = 0;2562	int cycled = 1;2563	handle_t *handle = NULL;2564	struct inode *inode = mpd->inode;2565	struct address_space *mapping = inode->i_mapping;2566	int needed_blocks, rsv_blocks = 0, ret = 0;2567	struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);2568	struct blk_plug plug;2569	bool give_up_on_write = false;2570 2571	trace_ext4_writepages(inode, wbc);2572 2573	/*2574	 * No pages to write? This is mainly a kludge to avoid starting2575	 * a transaction for special inodes like journal inode on last iput()2576	 * because that could violate lock ordering on umount2577	 */2578	if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))2579		goto out_writepages;2580 2581	/*2582	 * If the filesystem has aborted, it is read-only, so return2583	 * right away instead of dumping stack traces later on that2584	 * will obscure the real source of the problem.  We test2585	 * fs shutdown state instead of sb->s_flag's SB_RDONLY because2586	 * the latter could be true if the filesystem is mounted2587	 * read-only, and in that case, ext4_writepages should2588	 * *never* be called, so if that ever happens, we would want2589	 * the stack trace.2590	 */2591	if (unlikely(ext4_forced_shutdown(mapping->host->i_sb))) {2592		ret = -EROFS;2593		goto out_writepages;2594	}2595 2596	/*2597	 * If we have inline data and arrive here, it means that2598	 * we will soon create the block for the 1st page, so2599	 * we'd better clear the inline data here.2600	 */2601	if (ext4_has_inline_data(inode)) {2602		/* Just inode will be modified... */2603		handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);2604		if (IS_ERR(handle)) {2605			ret = PTR_ERR(handle);2606			goto out_writepages;2607		}2608		BUG_ON(ext4_test_inode_state(inode,2609				EXT4_STATE_MAY_INLINE_DATA));2610		ext4_destroy_inline_data(handle, inode);2611		ext4_journal_stop(handle);2612	}2613 2614	/*2615	 * data=journal mode does not do delalloc so we just need to writeout /2616	 * journal already mapped buffers. On the other hand we need to commit2617	 * transaction to make data stable. We expect all the data to be2618	 * already in the journal (the only exception are DMA pinned pages2619	 * dirtied behind our back) so we commit transaction here and run the2620	 * writeback loop to checkpoint them. The checkpointing is not actually2621	 * necessary to make data persistent *but* quite a few places (extent2622	 * shifting operations, fsverity, ...) depend on being able to drop2623	 * pagecache pages after calling filemap_write_and_wait() and for that2624	 * checkpointing needs to happen.2625	 */2626	if (ext4_should_journal_data(inode)) {2627		mpd->can_map = 0;2628		if (wbc->sync_mode == WB_SYNC_ALL)2629			ext4_fc_commit(sbi->s_journal,2630				       EXT4_I(inode)->i_datasync_tid);2631	}2632	mpd->journalled_more_data = 0;2633 2634	if (ext4_should_dioread_nolock(inode)) {2635		/*2636		 * We may need to convert up to one extent per block in2637		 * the page and we may dirty the inode.2638		 */2639		rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,2640						PAGE_SIZE >> inode->i_blkbits);2641	}2642 2643	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)2644		range_whole = 1;2645 2646	if (wbc->range_cyclic) {2647		writeback_index = mapping->writeback_index;2648		if (writeback_index)2649			cycled = 0;2650		mpd->first_page = writeback_index;2651		mpd->last_page = -1;2652	} else {2653		mpd->first_page = wbc->range_start >> PAGE_SHIFT;2654		mpd->last_page = wbc->range_end >> PAGE_SHIFT;2655	}2656 2657	ext4_io_submit_init(&mpd->io_submit, wbc);2658retry:2659	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)2660		tag_pages_for_writeback(mapping, mpd->first_page,2661					mpd->last_page);2662	blk_start_plug(&plug);2663 2664	/*2665	 * First writeback pages that don't need mapping - we can avoid2666	 * starting a transaction unnecessarily and also avoid being blocked2667	 * in the block layer on device congestion while having transaction2668	 * started.2669	 */2670	mpd->do_map = 0;2671	mpd->scanned_until_end = 0;2672	mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);2673	if (!mpd->io_submit.io_end) {2674		ret = -ENOMEM;2675		goto unplug;2676	}2677	ret = mpage_prepare_extent_to_map(mpd);2678	/* Unlock pages we didn't use */2679	mpage_release_unused_pages(mpd, false);2680	/* Submit prepared bio */2681	ext4_io_submit(&mpd->io_submit);2682	ext4_put_io_end_defer(mpd->io_submit.io_end);2683	mpd->io_submit.io_end = NULL;2684	if (ret < 0)2685		goto unplug;2686 2687	while (!mpd->scanned_until_end && wbc->nr_to_write > 0) {2688		/* For each extent of pages we use new io_end */2689		mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);2690		if (!mpd->io_submit.io_end) {2691			ret = -ENOMEM;2692			break;2693		}2694 2695		WARN_ON_ONCE(!mpd->can_map);2696		/*2697		 * We have two constraints: We find one extent to map and we2698		 * must always write out whole page (makes a difference when2699		 * blocksize < pagesize) so that we don't block on IO when we2700		 * try to write out the rest of the page. Journalled mode is2701		 * not supported by delalloc.2702		 */2703		BUG_ON(ext4_should_journal_data(inode));2704		needed_blocks = ext4_da_writepages_trans_blocks(inode);2705 2706		/* start a new transaction */2707		handle = ext4_journal_start_with_reserve(inode,2708				EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);2709		if (IS_ERR(handle)) {2710			ret = PTR_ERR(handle);2711			ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "2712			       "%ld pages, ino %lu; err %d", __func__,2713				wbc->nr_to_write, inode->i_ino, ret);2714			/* Release allocated io_end */2715			ext4_put_io_end(mpd->io_submit.io_end);2716			mpd->io_submit.io_end = NULL;2717			break;2718		}2719		mpd->do_map = 1;2720 2721		trace_ext4_da_write_pages(inode, mpd->first_page, wbc);2722		ret = mpage_prepare_extent_to_map(mpd);2723		if (!ret && mpd->map.m_len)2724			ret = mpage_map_and_submit_extent(handle, mpd,2725					&give_up_on_write);2726		/*2727		 * Caution: If the handle is synchronous,2728		 * ext4_journal_stop() can wait for transaction commit2729		 * to finish which may depend on writeback of pages to2730		 * complete or on page lock to be released.  In that2731		 * case, we have to wait until after we have2732		 * submitted all the IO, released page locks we hold,2733		 * and dropped io_end reference (for extent conversion2734		 * to be able to complete) before stopping the handle.2735		 */2736		if (!ext4_handle_valid(handle) || handle->h_sync == 0) {2737			ext4_journal_stop(handle);2738			handle = NULL;2739			mpd->do_map = 0;2740		}2741		/* Unlock pages we didn't use */2742		mpage_release_unused_pages(mpd, give_up_on_write);2743		/* Submit prepared bio */2744		ext4_io_submit(&mpd->io_submit);2745 2746		/*2747		 * Drop our io_end reference we got from init. We have2748		 * to be careful and use deferred io_end finishing if2749		 * we are still holding the transaction as we can2750		 * release the last reference to io_end which may end2751		 * up doing unwritten extent conversion.2752		 */2753		if (handle) {2754			ext4_put_io_end_defer(mpd->io_submit.io_end);2755			ext4_journal_stop(handle);2756		} else2757			ext4_put_io_end(mpd->io_submit.io_end);2758		mpd->io_submit.io_end = NULL;2759 2760		if (ret == -ENOSPC && sbi->s_journal) {2761			/*2762			 * Commit the transaction which would2763			 * free blocks released in the transaction2764			 * and try again2765			 */2766			jbd2_journal_force_commit_nested(sbi->s_journal);2767			ret = 0;2768			continue;2769		}2770		/* Fatal error - ENOMEM, EIO... */2771		if (ret)2772			break;2773	}2774unplug:2775	blk_finish_plug(&plug);2776	if (!ret && !cycled && wbc->nr_to_write > 0) {2777		cycled = 1;2778		mpd->last_page = writeback_index - 1;2779		mpd->first_page = 0;2780		goto retry;2781	}2782 2783	/* Update index */2784	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))2785		/*2786		 * Set the writeback_index so that range_cyclic2787		 * mode will write it back later2788		 */2789		mapping->writeback_index = mpd->first_page;2790 2791out_writepages:2792	trace_ext4_writepages_result(inode, wbc, ret,2793				     nr_to_write - wbc->nr_to_write);2794	return ret;2795}2796 2797static int ext4_writepages(struct address_space *mapping,2798			   struct writeback_control *wbc)2799{2800	struct super_block *sb = mapping->host->i_sb;2801	struct mpage_da_data mpd = {2802		.inode = mapping->host,2803		.wbc = wbc,2804		.can_map = 1,2805	};2806	int ret;2807	int alloc_ctx;2808 2809	if (unlikely(ext4_forced_shutdown(sb)))2810		return -EIO;2811 2812	alloc_ctx = ext4_writepages_down_read(sb);2813	ret = ext4_do_writepages(&mpd);2814	/*2815	 * For data=journal writeback we could have come across pages marked2816	 * for delayed dirtying (PageChecked) which were just added to the2817	 * running transaction. Try once more to get them to stable storage.2818	 */2819	if (!ret && mpd.journalled_more_data)2820		ret = ext4_do_writepages(&mpd);2821	ext4_writepages_up_read(sb, alloc_ctx);2822 2823	return ret;2824}2825 2826int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode)2827{2828	struct writeback_control wbc = {2829		.sync_mode = WB_SYNC_ALL,2830		.nr_to_write = LONG_MAX,2831		.range_start = jinode->i_dirty_start,2832		.range_end = jinode->i_dirty_end,2833	};2834	struct mpage_da_data mpd = {2835		.inode = jinode->i_vfs_inode,2836		.wbc = &wbc,2837		.can_map = 0,2838	};2839	return ext4_do_writepages(&mpd);2840}2841 2842static int ext4_dax_writepages(struct address_space *mapping,2843			       struct writeback_control *wbc)2844{2845	int ret;2846	long nr_to_write = wbc->nr_to_write;2847	struct inode *inode = mapping->host;2848	int alloc_ctx;2849 2850	if (unlikely(ext4_forced_shutdown(inode->i_sb)))2851		return -EIO;2852 2853	alloc_ctx = ext4_writepages_down_read(inode->i_sb);2854	trace_ext4_writepages(inode, wbc);2855 2856	ret = dax_writeback_mapping_range(mapping,2857					  EXT4_SB(inode->i_sb)->s_daxdev, wbc);2858	trace_ext4_writepages_result(inode, wbc, ret,2859				     nr_to_write - wbc->nr_to_write);2860	ext4_writepages_up_read(inode->i_sb, alloc_ctx);2861	return ret;2862}2863 2864static int ext4_nonda_switch(struct super_block *sb)2865{2866	s64 free_clusters, dirty_clusters;2867	struct ext4_sb_info *sbi = EXT4_SB(sb);2868 2869	/*2870	 * switch to non delalloc mode if we are running low2871	 * on free block. The free block accounting via percpu2872	 * counters can get slightly wrong with percpu_counter_batch getting2873	 * accumulated on each CPU without updating global counters2874	 * Delalloc need an accurate free block accounting. So switch2875	 * to non delalloc when we are near to error range.2876	 */2877	free_clusters =2878		percpu_counter_read_positive(&sbi->s_freeclusters_counter);2879	dirty_clusters =2880		percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);2881	/*2882	 * Start pushing delalloc when 1/2 of free blocks are dirty.2883	 */2884	if (dirty_clusters && (free_clusters < 2 * dirty_clusters))2885		try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);2886 2887	if (2 * free_clusters < 3 * dirty_clusters ||2888	    free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {2889		/*2890		 * free block count is less than 150% of dirty blocks2891		 * or free blocks is less than watermark2892		 */2893		return 1;2894	}2895	return 0;2896}2897 2898static int ext4_da_write_begin(struct file *file, struct address_space *mapping,2899			       loff_t pos, unsigned len,2900			       struct folio **foliop, void **fsdata)2901{2902	int ret, retries = 0;2903	struct folio *folio;2904	pgoff_t index;2905	struct inode *inode = mapping->host;2906 2907	if (unlikely(ext4_forced_shutdown(inode->i_sb)))2908		return -EIO;2909 2910	index = pos >> PAGE_SHIFT;2911 2912	if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) {2913		*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;2914		return ext4_write_begin(file, mapping, pos,2915					len, foliop, fsdata);2916	}2917	*fsdata = (void *)0;2918	trace_ext4_da_write_begin(inode, pos, len);2919 2920	if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {2921		ret = ext4_da_write_inline_data_begin(mapping, inode, pos, len,2922						      foliop, fsdata);2923		if (ret < 0)2924			return ret;2925		if (ret == 1)2926			return 0;2927	}2928 2929retry:2930	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,2931			mapping_gfp_mask(mapping));2932	if (IS_ERR(folio))2933		return PTR_ERR(folio);2934 2935	ret = ext4_block_write_begin(NULL, folio, pos, len,2936				     ext4_da_get_block_prep);2937	if (ret < 0) {2938		folio_unlock(folio);2939		folio_put(folio);2940		/*2941		 * block_write_begin may have instantiated a few blocks2942		 * outside i_size.  Trim these off again. Don't need2943		 * i_size_read because we hold inode lock.2944		 */2945		if (pos + len > inode->i_size)2946			ext4_truncate_failed_write(inode);2947 2948		if (ret == -ENOSPC &&2949		    ext4_should_retry_alloc(inode->i_sb, &retries))2950			goto retry;2951		return ret;2952	}2953 2954	*foliop = folio;2955	return ret;2956}2957 2958/*2959 * Check if we should update i_disksize2960 * when write to the end of file but not require block allocation2961 */2962static int ext4_da_should_update_i_disksize(struct folio *folio,2963					    unsigned long offset)2964{2965	struct buffer_head *bh;2966	struct inode *inode = folio->mapping->host;2967	unsigned int idx;2968	int i;2969 2970	bh = folio_buffers(folio);2971	idx = offset >> inode->i_blkbits;2972 2973	for (i = 0; i < idx; i++)2974		bh = bh->b_this_page;2975 2976	if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))2977		return 0;2978	return 1;2979}2980 2981static int ext4_da_do_write_end(struct address_space *mapping,2982			loff_t pos, unsigned len, unsigned copied,2983			struct folio *folio)2984{2985	struct inode *inode = mapping->host;2986	loff_t old_size = inode->i_size;2987	bool disksize_changed = false;2988	loff_t new_i_size;2989 2990	if (unlikely(!folio_buffers(folio))) {2991		folio_unlock(folio);2992		folio_put(folio);2993		return -EIO;2994	}2995	/*2996	 * block_write_end() will mark the inode as dirty with I_DIRTY_PAGES2997	 * flag, which all that's needed to trigger page writeback.2998	 */2999	copied = block_write_end(NULL, mapping, pos, len, copied,3000			folio, NULL);3001	new_i_size = pos + copied;3002 3003	/*3004	 * It's important to update i_size while still holding folio lock,3005	 * because folio writeout could otherwise come in and zero beyond3006	 * i_size.3007	 *3008	 * Since we are holding inode lock, we are sure i_disksize <=3009	 * i_size. We also know that if i_disksize < i_size, there are3010	 * delalloc writes pending in the range up to i_size. If the end of3011	 * the current write is <= i_size, there's no need to touch3012	 * i_disksize since writeback will push i_disksize up to i_size3013	 * eventually. If the end of the current write is > i_size and3014	 * inside an allocated block which ext4_da_should_update_i_disksize()3015	 * checked, we need to update i_disksize here as certain3016	 * ext4_writepages() paths not allocating blocks and update i_disksize.3017	 */3018	if (new_i_size > inode->i_size) {3019		unsigned long end;3020 3021		i_size_write(inode, new_i_size);3022		end = (new_i_size - 1) & (PAGE_SIZE - 1);3023		if (copied && ext4_da_should_update_i_disksize(folio, end)) {3024			ext4_update_i_disksize(inode, new_i_size);3025			disksize_changed = true;3026		}3027	}3028 3029	folio_unlock(folio);3030	folio_put(folio);3031 3032	if (old_size < pos)3033		pagecache_isize_extended(inode, old_size, pos);3034 3035	if (disksize_changed) {3036		handle_t *handle;3037 3038		handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);3039		if (IS_ERR(handle))3040			return PTR_ERR(handle);3041		ext4_mark_inode_dirty(handle, inode);3042		ext4_journal_stop(handle);3043	}3044 3045	return copied;3046}3047 3048static int ext4_da_write_end(struct file *file,3049			     struct address_space *mapping,3050			     loff_t pos, unsigned len, unsigned copied,3051			     struct folio *folio, void *fsdata)3052{3053	struct inode *inode = mapping->host;3054	int write_mode = (int)(unsigned long)fsdata;3055 3056	if (write_mode == FALL_BACK_TO_NONDELALLOC)3057		return ext4_write_end(file, mapping, pos,3058				      len, copied, folio, fsdata);3059 3060	trace_ext4_da_write_end(inode, pos, len, copied);3061 3062	if (write_mode != CONVERT_INLINE_DATA &&3063	    ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&3064	    ext4_has_inline_data(inode))3065		return ext4_write_inline_data_end(inode, pos, len, copied,3066						  folio);3067 3068	if (unlikely(copied < len) && !folio_test_uptodate(folio))3069		copied = 0;3070 3071	return ext4_da_do_write_end(mapping, pos, len, copied, folio);3072}3073 3074/*3075 * Force all delayed allocation blocks to be allocated for a given inode.3076 */3077int ext4_alloc_da_blocks(struct inode *inode)3078{3079	trace_ext4_alloc_da_blocks(inode);3080 3081	if (!EXT4_I(inode)->i_reserved_data_blocks)3082		return 0;3083 3084	/*3085	 * We do something simple for now.  The filemap_flush() will3086	 * also start triggering a write of the data blocks, which is3087	 * not strictly speaking necessary (and for users of3088	 * laptop_mode, not even desirable).  However, to do otherwise3089	 * would require replicating code paths in:3090	 *3091	 * ext4_writepages() ->3092	 *    write_cache_pages() ---> (via passed in callback function)3093	 *        __mpage_da_writepage() -->3094	 *           mpage_add_bh_to_extent()3095	 *           mpage_da_map_blocks()3096	 *3097	 * The problem is that write_cache_pages(), located in3098	 * mm/page-writeback.c, marks pages clean in preparation for3099	 * doing I/O, which is not desirable if we're not planning on3100	 * doing I/O at all.3101	 *3102	 * We could call write_cache_pages(), and then redirty all of3103	 * the pages by calling redirty_page_for_writepage() but that3104	 * would be ugly in the extreme.  So instead we would need to3105	 * replicate parts of the code in the above functions,3106	 * simplifying them because we wouldn't actually intend to3107	 * write out the pages, but rather only collect contiguous3108	 * logical block extents, call the multi-block allocator, and3109	 * then update the buffer heads with the block allocations.3110	 *3111	 * For now, though, we'll cheat by calling filemap_flush(),3112	 * which will map the blocks, and start the I/O, but not3113	 * actually wait for the I/O to complete.3114	 */3115	return filemap_flush(inode->i_mapping);3116}3117 3118/*3119 * bmap() is special.  It gets used by applications such as lilo and by3120 * the swapper to find the on-disk block of a specific piece of data.3121 *3122 * Naturally, this is dangerous if the block concerned is still in the3123 * journal.  If somebody makes a swapfile on an ext4 data-journaling3124 * filesystem and enables swap, then they may get a nasty shock when the3125 * data getting swapped to that swapfile suddenly gets overwritten by3126 * the original zero's written out previously to the journal and3127 * awaiting writeback in the kernel's buffer cache.3128 *3129 * So, if we see any bmap calls here on a modified, data-journaled file,3130 * take extra steps to flush any blocks which might be in the cache.3131 */3132static sector_t ext4_bmap(struct address_space *mapping, sector_t block)3133{3134	struct inode *inode = mapping->host;3135	sector_t ret = 0;3136 3137	inode_lock_shared(inode);3138	/*3139	 * We can get here for an inline file via the FIBMAP ioctl3140	 */3141	if (ext4_has_inline_data(inode))3142		goto out;3143 3144	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&3145	    (test_opt(inode->i_sb, DELALLOC) ||3146	     ext4_should_journal_data(inode))) {3147		/*3148		 * With delalloc or journalled data we want to sync the file so3149		 * that we can make sure we allocate blocks for file and data3150		 * is in place for the user to see it3151		 */3152		filemap_write_and_wait(mapping);3153	}3154 3155	ret = iomap_bmap(mapping, block, &ext4_iomap_ops);3156 3157out:3158	inode_unlock_shared(inode);3159	return ret;3160}3161 3162static int ext4_read_folio(struct file *file, struct folio *folio)3163{3164	int ret = -EAGAIN;3165	struct inode *inode = folio->mapping->host;3166 3167	trace_ext4_read_folio(inode, folio);3168 3169	if (ext4_has_inline_data(inode))3170		ret = ext4_readpage_inline(inode, folio);3171 3172	if (ret == -EAGAIN)3173		return ext4_mpage_readpages(inode, NULL, folio);3174 3175	return ret;3176}3177 3178static void ext4_readahead(struct readahead_control *rac)3179{3180	struct inode *inode = rac->mapping->host;3181 3182	/* If the file has inline data, no need to do readahead. */3183	if (ext4_has_inline_data(inode))3184		return;3185 3186	ext4_mpage_readpages(inode, rac, NULL);3187}3188 3189static void ext4_invalidate_folio(struct folio *folio, size_t offset,3190				size_t length)3191{3192	trace_ext4_invalidate_folio(folio, offset, length);3193 3194	/* No journalling happens on data buffers when this function is used */3195	WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio)));3196 3197	block_invalidate_folio(folio, offset, length);3198}3199 3200static int __ext4_journalled_invalidate_folio(struct folio *folio,3201					    size_t offset, size_t length)3202{3203	journal_t *journal = EXT4_JOURNAL(folio->mapping->host);3204 3205	trace_ext4_journalled_invalidate_folio(folio, offset, length);3206 3207	/*3208	 * If it's a full truncate we just forget about the pending dirtying3209	 */3210	if (offset == 0 && length == folio_size(folio))3211		folio_clear_checked(folio);3212 3213	return jbd2_journal_invalidate_folio(journal, folio, offset, length);3214}3215 3216/* Wrapper for aops... */3217static void ext4_journalled_invalidate_folio(struct folio *folio,3218					   size_t offset,3219					   size_t length)3220{3221	WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0);3222}3223 3224static bool ext4_release_folio(struct folio *folio, gfp_t wait)3225{3226	struct inode *inode = folio->mapping->host;3227	journal_t *journal = EXT4_JOURNAL(inode);3228 3229	trace_ext4_release_folio(inode, folio);3230 3231	/* Page has dirty journalled data -> cannot release */3232	if (folio_test_checked(folio))3233		return false;3234	if (journal)3235		return jbd2_journal_try_to_free_buffers(journal, folio);3236	else3237		return try_to_free_buffers(folio);3238}3239 3240static bool ext4_inode_datasync_dirty(struct inode *inode)3241{3242	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;3243 3244	if (journal) {3245		if (jbd2_transaction_committed(journal,3246			EXT4_I(inode)->i_datasync_tid))3247			return false;3248		if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))3249			return !list_empty(&EXT4_I(inode)->i_fc_list);3250		return true;3251	}3252 3253	/* Any metadata buffers to write? */3254	if (!list_empty(&inode->i_mapping->i_private_list))3255		return true;3256	return inode->i_state & I_DIRTY_DATASYNC;3257}3258 3259static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,3260			   struct ext4_map_blocks *map, loff_t offset,3261			   loff_t length, unsigned int flags)3262{3263	u8 blkbits = inode->i_blkbits;3264 3265	/*3266	 * Writes that span EOF might trigger an I/O size update on completion,3267	 * so consider them to be dirty for the purpose of O_DSYNC, even if3268	 * there is no other metadata changes being made or are pending.3269	 */3270	iomap->flags = 0;3271	if (ext4_inode_datasync_dirty(inode) ||3272	    offset + length > i_size_read(inode))3273		iomap->flags |= IOMAP_F_DIRTY;3274 3275	if (map->m_flags & EXT4_MAP_NEW)3276		iomap->flags |= IOMAP_F_NEW;3277 3278	if (flags & IOMAP_DAX)3279		iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;3280	else3281		iomap->bdev = inode->i_sb->s_bdev;3282	iomap->offset = (u64) map->m_lblk << blkbits;3283	iomap->length = (u64) map->m_len << blkbits;3284 3285	if ((map->m_flags & EXT4_MAP_MAPPED) &&3286	    !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))3287		iomap->flags |= IOMAP_F_MERGED;3288 3289	/*3290	 * Flags passed to ext4_map_blocks() for direct I/O writes can result3291	 * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits3292	 * set. In order for any allocated unwritten extents to be converted3293	 * into written extents correctly within the ->end_io() handler, we3294	 * need to ensure that the iomap->type is set appropriately. Hence, the3295	 * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has3296	 * been set first.3297	 */3298	if (map->m_flags & EXT4_MAP_UNWRITTEN) {3299		iomap->type = IOMAP_UNWRITTEN;3300		iomap->addr = (u64) map->m_pblk << blkbits;3301		if (flags & IOMAP_DAX)3302			iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;3303	} else if (map->m_flags & EXT4_MAP_MAPPED) {3304		iomap->type = IOMAP_MAPPED;3305		iomap->addr = (u64) map->m_pblk << blkbits;3306		if (flags & IOMAP_DAX)3307			iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;3308	} else if (map->m_flags & EXT4_MAP_DELAYED) {3309		iomap->type = IOMAP_DELALLOC;3310		iomap->addr = IOMAP_NULL_ADDR;3311	} else {3312		iomap->type = IOMAP_HOLE;3313		iomap->addr = IOMAP_NULL_ADDR;3314	}3315}3316 3317static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,3318			    unsigned int flags)3319{3320	handle_t *handle;3321	u8 blkbits = inode->i_blkbits;3322	int ret, dio_credits, m_flags = 0, retries = 0;3323 3324	/*3325	 * Trim the mapping request to the maximum value that we can map at3326	 * once for direct I/O.3327	 */3328	if (map->m_len > DIO_MAX_BLOCKS)3329		map->m_len = DIO_MAX_BLOCKS;3330	dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);3331 3332retry:3333	/*3334	 * Either we allocate blocks and then don't get an unwritten extent, so3335	 * in that case we have reserved enough credits. Or, the blocks are3336	 * already allocated and unwritten. In that case, the extent conversion3337	 * fits into the credits as well.3338	 */3339	handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);3340	if (IS_ERR(handle))3341		return PTR_ERR(handle);3342 3343	/*3344	 * DAX and direct I/O are the only two operations that are currently3345	 * supported with IOMAP_WRITE.3346	 */3347	WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT)));3348	if (flags & IOMAP_DAX)3349		m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;3350	/*3351	 * We use i_size instead of i_disksize here because delalloc writeback3352	 * can complete at any point during the I/O and subsequently push the3353	 * i_disksize out to i_size. This could be beyond where direct I/O is3354	 * happening and thus expose allocated blocks to direct I/O reads.3355	 */3356	else if (((loff_t)map->m_lblk << blkbits) >= i_size_read(inode))3357		m_flags = EXT4_GET_BLOCKS_CREATE;3358	else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))3359		m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;3360 3361	ret = ext4_map_blocks(handle, inode, map, m_flags);3362 3363	/*3364	 * We cannot fill holes in indirect tree based inodes as that could3365	 * expose stale data in the case of a crash. Use the magic error code3366	 * to fallback to buffered I/O.3367	 */3368	if (!m_flags && !ret)3369		ret = -ENOTBLK;3370 3371	ext4_journal_stop(handle);3372	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))3373		goto retry;3374 3375	return ret;3376}3377 3378 3379static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,3380		unsigned flags, struct iomap *iomap, struct iomap *srcmap)3381{3382	int ret;3383	struct ext4_map_blocks map;3384	u8 blkbits = inode->i_blkbits;3385 3386	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)3387		return -EINVAL;3388 3389	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))3390		return -ERANGE;3391 3392	/*3393	 * Calculate the first and last logical blocks respectively.3394	 */3395	map.m_lblk = offset >> blkbits;3396	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,3397			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;3398 3399	if (flags & IOMAP_WRITE) {3400		/*3401		 * We check here if the blocks are already allocated, then we3402		 * don't need to start a journal txn and we can directly return3403		 * the mapping information. This could boost performance3404		 * especially in multi-threaded overwrite requests.3405		 */3406		if (offset + length <= i_size_read(inode)) {3407			ret = ext4_map_blocks(NULL, inode, &map, 0);3408			if (ret > 0 && (map.m_flags & EXT4_MAP_MAPPED))3409				goto out;3410		}3411		ret = ext4_iomap_alloc(inode, &map, flags);3412	} else {3413		ret = ext4_map_blocks(NULL, inode, &map, 0);3414	}3415 3416	if (ret < 0)3417		return ret;3418out:3419	/*3420	 * When inline encryption is enabled, sometimes I/O to an encrypted file3421	 * has to be broken up to guarantee DUN contiguity.  Handle this by3422	 * limiting the length of the mapping returned.3423	 */3424	map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);3425 3426	ext4_set_iomap(inode, iomap, &map, offset, length, flags);3427 3428	return 0;3429}3430 3431static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,3432		loff_t length, unsigned flags, struct iomap *iomap,3433		struct iomap *srcmap)3434{3435	int ret;3436 3437	/*3438	 * Even for writes we don't need to allocate blocks, so just pretend3439	 * we are reading to save overhead of starting a transaction.3440	 */3441	flags &= ~IOMAP_WRITE;3442	ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);3443	WARN_ON_ONCE(!ret && iomap->type != IOMAP_MAPPED);3444	return ret;3445}3446 3447static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,3448			  ssize_t written, unsigned flags, struct iomap *iomap)3449{3450	/*3451	 * Check to see whether an error occurred while writing out the data to3452	 * the allocated blocks. If so, return the magic error code so that we3453	 * fallback to buffered I/O and attempt to complete the remainder of3454	 * the I/O. Any blocks that may have been allocated in preparation for3455	 * the direct I/O will be reused during buffered I/O.3456	 */3457	if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0)3458		return -ENOTBLK;3459 3460	return 0;3461}3462 3463const struct iomap_ops ext4_iomap_ops = {3464	.iomap_begin		= ext4_iomap_begin,3465	.iomap_end		= ext4_iomap_end,3466};3467 3468const struct iomap_ops ext4_iomap_overwrite_ops = {3469	.iomap_begin		= ext4_iomap_overwrite_begin,3470	.iomap_end		= ext4_iomap_end,3471};3472 3473static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,3474				   loff_t length, unsigned int flags,3475				   struct iomap *iomap, struct iomap *srcmap)3476{3477	int ret;3478	struct ext4_map_blocks map;3479	u8 blkbits = inode->i_blkbits;3480 3481	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)3482		return -EINVAL;3483 3484	if (ext4_has_inline_data(inode)) {3485		ret = ext4_inline_data_iomap(inode, iomap);3486		if (ret != -EAGAIN) {3487			if (ret == 0 && offset >= iomap->length)3488				ret = -ENOENT;3489			return ret;3490		}3491	}3492 3493	/*3494	 * Calculate the first and last logical block respectively.3495	 */3496	map.m_lblk = offset >> blkbits;3497	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,3498			  EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;3499 3500	/*3501	 * Fiemap callers may call for offset beyond s_bitmap_maxbytes.3502	 * So handle it here itself instead of querying ext4_map_blocks().3503	 * Since ext4_map_blocks() will warn about it and will return3504	 * -EIO error.3505	 */3506	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {3507		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);3508 3509		if (offset >= sbi->s_bitmap_maxbytes) {3510			map.m_flags = 0;3511			goto set_iomap;3512		}3513	}3514 3515	ret = ext4_map_blocks(NULL, inode, &map, 0);3516	if (ret < 0)3517		return ret;3518set_iomap:3519	ext4_set_iomap(inode, iomap, &map, offset, length, flags);3520 3521	return 0;3522}3523 3524const struct iomap_ops ext4_iomap_report_ops = {3525	.iomap_begin = ext4_iomap_begin_report,3526};3527 3528/*3529 * For data=journal mode, folio should be marked dirty only when it was3530 * writeably mapped. When that happens, it was already attached to the3531 * transaction and marked as jbddirty (we take care of this in3532 * ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings3533 * so we should have nothing to do here, except for the case when someone3534 * had the page pinned and dirtied the page through this pin (e.g. by doing3535 * direct IO to it). In that case we'd need to attach buffers here to the3536 * transaction but we cannot due to lock ordering.  We cannot just dirty the3537 * folio and leave attached buffers clean, because the buffers' dirty state is3538 * "definitive".  We cannot just set the buffers dirty or jbddirty because all3539 * the journalling code will explode.  So what we do is to mark the folio3540 * "pending dirty" and next time ext4_writepages() is called, attach buffers3541 * to the transaction appropriately.3542 */3543static bool ext4_journalled_dirty_folio(struct address_space *mapping,3544		struct folio *folio)3545{3546	WARN_ON_ONCE(!folio_buffers(folio));3547	if (folio_maybe_dma_pinned(folio))3548		folio_set_checked(folio);3549	return filemap_dirty_folio(mapping, folio);3550}3551 3552static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio)3553{3554	WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio));3555	WARN_ON_ONCE(!folio_buffers(folio));3556	return block_dirty_folio(mapping, folio);3557}3558 3559static int ext4_iomap_swap_activate(struct swap_info_struct *sis,3560				    struct file *file, sector_t *span)3561{3562	return iomap_swapfile_activate(sis, file, span,3563				       &ext4_iomap_report_ops);3564}3565 3566static const struct address_space_operations ext4_aops = {3567	.read_folio		= ext4_read_folio,3568	.readahead		= ext4_readahead,3569	.writepages		= ext4_writepages,3570	.write_begin		= ext4_write_begin,3571	.write_end		= ext4_write_end,3572	.dirty_folio		= ext4_dirty_folio,3573	.bmap			= ext4_bmap,3574	.invalidate_folio	= ext4_invalidate_folio,3575	.release_folio		= ext4_release_folio,3576	.migrate_folio		= buffer_migrate_folio,3577	.is_partially_uptodate  = block_is_partially_uptodate,3578	.error_remove_folio	= generic_error_remove_folio,3579	.swap_activate		= ext4_iomap_swap_activate,3580};3581 3582static const struct address_space_operations ext4_journalled_aops = {3583	.read_folio		= ext4_read_folio,3584	.readahead		= ext4_readahead,3585	.writepages		= ext4_writepages,3586	.write_begin		= ext4_write_begin,3587	.write_end		= ext4_journalled_write_end,3588	.dirty_folio		= ext4_journalled_dirty_folio,3589	.bmap			= ext4_bmap,3590	.invalidate_folio	= ext4_journalled_invalidate_folio,3591	.release_folio		= ext4_release_folio,3592	.migrate_folio		= buffer_migrate_folio_norefs,3593	.is_partially_uptodate  = block_is_partially_uptodate,3594	.error_remove_folio	= generic_error_remove_folio,3595	.swap_activate		= ext4_iomap_swap_activate,3596};3597 3598static const struct address_space_operations ext4_da_aops = {3599	.read_folio		= ext4_read_folio,3600	.readahead		= ext4_readahead,3601	.writepages		= ext4_writepages,3602	.write_begin		= ext4_da_write_begin,3603	.write_end		= ext4_da_write_end,3604	.dirty_folio		= ext4_dirty_folio,3605	.bmap			= ext4_bmap,3606	.invalidate_folio	= ext4_invalidate_folio,3607	.release_folio		= ext4_release_folio,3608	.migrate_folio		= buffer_migrate_folio,3609	.is_partially_uptodate  = block_is_partially_uptodate,3610	.error_remove_folio	= generic_error_remove_folio,3611	.swap_activate		= ext4_iomap_swap_activate,3612};3613 3614static const struct address_space_operations ext4_dax_aops = {3615	.writepages		= ext4_dax_writepages,3616	.dirty_folio		= noop_dirty_folio,3617	.bmap			= ext4_bmap,3618	.swap_activate		= ext4_iomap_swap_activate,3619};3620 3621void ext4_set_aops(struct inode *inode)3622{3623	switch (ext4_inode_journal_mode(inode)) {3624	case EXT4_INODE_ORDERED_DATA_MODE:3625	case EXT4_INODE_WRITEBACK_DATA_MODE:3626		break;3627	case EXT4_INODE_JOURNAL_DATA_MODE:3628		inode->i_mapping->a_ops = &ext4_journalled_aops;3629		return;3630	default:3631		BUG();3632	}3633	if (IS_DAX(inode))3634		inode->i_mapping->a_ops = &ext4_dax_aops;3635	else if (test_opt(inode->i_sb, DELALLOC))3636		inode->i_mapping->a_ops = &ext4_da_aops;3637	else3638		inode->i_mapping->a_ops = &ext4_aops;3639}3640 3641/*3642 * Here we can't skip an unwritten buffer even though it usually reads zero3643 * because it might have data in pagecache (eg, if called from ext4_zero_range,3644 * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a3645 * racing writeback can come later and flush the stale pagecache to disk.3646 */3647static int __ext4_block_zero_page_range(handle_t *handle,3648		struct address_space *mapping, loff_t from, loff_t length)3649{3650	ext4_fsblk_t index = from >> PAGE_SHIFT;3651	unsigned offset = from & (PAGE_SIZE-1);3652	unsigned blocksize, pos;3653	ext4_lblk_t iblock;3654	struct inode *inode = mapping->host;3655	struct buffer_head *bh;3656	struct folio *folio;3657	int err = 0;3658 3659	folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT,3660				    FGP_LOCK | FGP_ACCESSED | FGP_CREAT,3661				    mapping_gfp_constraint(mapping, ~__GFP_FS));3662	if (IS_ERR(folio))3663		return PTR_ERR(folio);3664 3665	blocksize = inode->i_sb->s_blocksize;3666 3667	iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);3668 3669	bh = folio_buffers(folio);3670	if (!bh)3671		bh = create_empty_buffers(folio, blocksize, 0);3672 3673	/* Find the buffer that contains "offset" */3674	pos = blocksize;3675	while (offset >= pos) {3676		bh = bh->b_this_page;3677		iblock++;3678		pos += blocksize;3679	}3680	if (buffer_freed(bh)) {3681		BUFFER_TRACE(bh, "freed: skip");3682		goto unlock;3683	}3684	if (!buffer_mapped(bh)) {3685		BUFFER_TRACE(bh, "unmapped");3686		ext4_get_block(inode, iblock, bh, 0);3687		/* unmapped? It's a hole - nothing to do */3688		if (!buffer_mapped(bh)) {3689			BUFFER_TRACE(bh, "still unmapped");3690			goto unlock;3691		}3692	}3693 3694	/* Ok, it's mapped. Make sure it's up-to-date */3695	if (folio_test_uptodate(folio))3696		set_buffer_uptodate(bh);3697 3698	if (!buffer_uptodate(bh)) {3699		err = ext4_read_bh_lock(bh, 0, true);3700		if (err)3701			goto unlock;3702		if (fscrypt_inode_uses_fs_layer_crypto(inode)) {3703			/* We expect the key to be set. */3704			BUG_ON(!fscrypt_has_encryption_key(inode));3705			err = fscrypt_decrypt_pagecache_blocks(folio,3706							       blocksize,3707							       bh_offset(bh));3708			if (err) {3709				clear_buffer_uptodate(bh);3710				goto unlock;3711			}3712		}3713	}3714	if (ext4_should_journal_data(inode)) {3715		BUFFER_TRACE(bh, "get write access");3716		err = ext4_journal_get_write_access(handle, inode->i_sb, bh,3717						    EXT4_JTR_NONE);3718		if (err)3719			goto unlock;3720	}3721	folio_zero_range(folio, offset, length);3722	BUFFER_TRACE(bh, "zeroed end of block");3723 3724	if (ext4_should_journal_data(inode)) {3725		err = ext4_dirty_journalled_data(handle, bh);3726	} else {3727		err = 0;3728		mark_buffer_dirty(bh);3729		if (ext4_should_order_data(inode))3730			err = ext4_jbd2_inode_add_write(handle, inode, from,3731					length);3732	}3733 3734unlock:3735	folio_unlock(folio);3736	folio_put(folio);3737	return err;3738}3739 3740/*3741 * ext4_block_zero_page_range() zeros out a mapping of length 'length'3742 * starting from file offset 'from'.  The range to be zero'd must3743 * be contained with in one block.  If the specified range exceeds3744 * the end of the block it will be shortened to end of the block3745 * that corresponds to 'from'3746 */3747static int ext4_block_zero_page_range(handle_t *handle,3748		struct address_space *mapping, loff_t from, loff_t length)3749{3750	struct inode *inode = mapping->host;3751	unsigned offset = from & (PAGE_SIZE-1);3752	unsigned blocksize = inode->i_sb->s_blocksize;3753	unsigned max = blocksize - (offset & (blocksize - 1));3754 3755	/*3756	 * correct length if it does not fall between3757	 * 'from' and the end of the block3758	 */3759	if (length > max || length < 0)3760		length = max;3761 3762	if (IS_DAX(inode)) {3763		return dax_zero_range(inode, from, length, NULL,3764				      &ext4_iomap_ops);3765	}3766	return __ext4_block_zero_page_range(handle, mapping, from, length);3767}3768 3769/*3770 * ext4_block_truncate_page() zeroes out a mapping from file offset `from'3771 * up to the end of the block which corresponds to `from'.3772 * This required during truncate. We need to physically zero the tail end3773 * of that block so it doesn't yield old data if the file is later grown.3774 */3775static int ext4_block_truncate_page(handle_t *handle,3776		struct address_space *mapping, loff_t from)3777{3778	unsigned offset = from & (PAGE_SIZE-1);3779	unsigned length;3780	unsigned blocksize;3781	struct inode *inode = mapping->host;3782 3783	/* If we are processing an encrypted inode during orphan list handling */3784	if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))3785		return 0;3786 3787	blocksize = inode->i_sb->s_blocksize;3788	length = blocksize - (offset & (blocksize - 1));3789 3790	return ext4_block_zero_page_range(handle, mapping, from, length);3791}3792 3793int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,3794			     loff_t lstart, loff_t length)3795{3796	struct super_block *sb = inode->i_sb;3797	struct address_space *mapping = inode->i_mapping;3798	unsigned partial_start, partial_end;3799	ext4_fsblk_t start, end;3800	loff_t byte_end = (lstart + length - 1);3801	int err = 0;3802 3803	partial_start = lstart & (sb->s_blocksize - 1);3804	partial_end = byte_end & (sb->s_blocksize - 1);3805 3806	start = lstart >> sb->s_blocksize_bits;3807	end = byte_end >> sb->s_blocksize_bits;3808 3809	/* Handle partial zero within the single block */3810	if (start == end &&3811	    (partial_start || (partial_end != sb->s_blocksize - 1))) {3812		err = ext4_block_zero_page_range(handle, mapping,3813						 lstart, length);3814		return err;3815	}3816	/* Handle partial zero out on the start of the range */3817	if (partial_start) {3818		err = ext4_block_zero_page_range(handle, mapping,3819						 lstart, sb->s_blocksize);3820		if (err)3821			return err;3822	}3823	/* Handle partial zero out on the end of the range */3824	if (partial_end != sb->s_blocksize - 1)3825		err = ext4_block_zero_page_range(handle, mapping,3826						 byte_end - partial_end,3827						 partial_end + 1);3828	return err;3829}3830 3831int ext4_can_truncate(struct inode *inode)3832{3833	if (S_ISREG(inode->i_mode))3834		return 1;3835	if (S_ISDIR(inode->i_mode))3836		return 1;3837	if (S_ISLNK(inode->i_mode))3838		return !ext4_inode_is_fast_symlink(inode);3839	return 0;3840}3841 3842/*3843 * We have to make sure i_disksize gets properly updated before we truncate3844 * page cache due to hole punching or zero range. Otherwise i_disksize update3845 * can get lost as it may have been postponed to submission of writeback but3846 * that will never happen after we truncate page cache.3847 */3848int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,3849				      loff_t len)3850{3851	handle_t *handle;3852	int ret;3853 3854	loff_t size = i_size_read(inode);3855 3856	WARN_ON(!inode_is_locked(inode));3857	if (offset > size || offset + len < size)3858		return 0;3859 3860	if (EXT4_I(inode)->i_disksize >= size)3861		return 0;3862 3863	handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);3864	if (IS_ERR(handle))3865		return PTR_ERR(handle);3866	ext4_update_i_disksize(inode, size);3867	ret = ext4_mark_inode_dirty(handle, inode);3868	ext4_journal_stop(handle);3869 3870	return ret;3871}3872 3873static void ext4_wait_dax_page(struct inode *inode)3874{3875	filemap_invalidate_unlock(inode->i_mapping);3876	schedule();3877	filemap_invalidate_lock(inode->i_mapping);3878}3879 3880int ext4_break_layouts(struct inode *inode)3881{3882	struct page *page;3883	int error;3884 3885	if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))3886		return -EINVAL;3887 3888	do {3889		page = dax_layout_busy_page(inode->i_mapping);3890		if (!page)3891			return 0;3892 3893		error = ___wait_var_event(&page->_refcount,3894				atomic_read(&page->_refcount) == 1,3895				TASK_INTERRUPTIBLE, 0, 0,3896				ext4_wait_dax_page(inode));3897	} while (error == 0);3898 3899	return error;3900}3901 3902/*3903 * ext4_punch_hole: punches a hole in a file by releasing the blocks3904 * associated with the given offset and length3905 *3906 * @inode:  File inode3907 * @offset: The offset where the hole will begin3908 * @len:    The length of the hole3909 *3910 * Returns: 0 on success or negative on failure3911 */3912 3913int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)3914{3915	struct inode *inode = file_inode(file);3916	struct super_block *sb = inode->i_sb;3917	ext4_lblk_t first_block, stop_block;3918	struct address_space *mapping = inode->i_mapping;3919	loff_t first_block_offset, last_block_offset, max_length;3920	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);3921	handle_t *handle;3922	unsigned int credits;3923	int ret = 0, ret2 = 0;3924 3925	trace_ext4_punch_hole(inode, offset, length, 0);3926 3927	/*3928	 * Write out all dirty pages to avoid race conditions3929	 * Then release them.3930	 */3931	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {3932		ret = filemap_write_and_wait_range(mapping, offset,3933						   offset + length - 1);3934		if (ret)3935			return ret;3936	}3937 3938	inode_lock(inode);3939 3940	/* No need to punch hole beyond i_size */3941	if (offset >= inode->i_size)3942		goto out_mutex;3943 3944	/*3945	 * If the hole extends beyond i_size, set the hole3946	 * to end after the page that contains i_size3947	 */3948	if (offset + length > inode->i_size) {3949		length = inode->i_size +3950		   PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -3951		   offset;3952	}3953 3954	/*3955	 * For punch hole the length + offset needs to be within one block3956	 * before last range. Adjust the length if it goes beyond that limit.3957	 */3958	max_length = sbi->s_bitmap_maxbytes - inode->i_sb->s_blocksize;3959	if (offset + length > max_length)3960		length = max_length - offset;3961 3962	if (offset & (sb->s_blocksize - 1) ||3963	    (offset + length) & (sb->s_blocksize - 1)) {3964		/*3965		 * Attach jinode to inode for jbd2 if we do any zeroing of3966		 * partial block3967		 */3968		ret = ext4_inode_attach_jinode(inode);3969		if (ret < 0)3970			goto out_mutex;3971 3972	}3973 3974	/* Wait all existing dio workers, newcomers will block on i_rwsem */3975	inode_dio_wait(inode);3976 3977	ret = file_modified(file);3978	if (ret)3979		goto out_mutex;3980 3981	/*3982	 * Prevent page faults from reinstantiating pages we have released from3983	 * page cache.3984	 */3985	filemap_invalidate_lock(mapping);3986 3987	ret = ext4_break_layouts(inode);3988	if (ret)3989		goto out_dio;3990 3991	first_block_offset = round_up(offset, sb->s_blocksize);3992	last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;3993 3994	/* Now release the pages and zero block aligned part of pages*/3995	if (last_block_offset > first_block_offset) {3996		ret = ext4_update_disksize_before_punch(inode, offset, length);3997		if (ret)3998			goto out_dio;3999		truncate_pagecache_range(inode, first_block_offset,4000					 last_block_offset);4001	}4002 4003	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))4004		credits = ext4_writepage_trans_blocks(inode);4005	else4006		credits = ext4_blocks_for_truncate(inode);4007	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);4008	if (IS_ERR(handle)) {4009		ret = PTR_ERR(handle);4010		ext4_std_error(sb, ret);4011		goto out_dio;4012	}4013 4014	ret = ext4_zero_partial_blocks(handle, inode, offset,4015				       length);4016	if (ret)4017		goto out_stop;4018 4019	first_block = (offset + sb->s_blocksize - 1) >>4020		EXT4_BLOCK_SIZE_BITS(sb);4021	stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);4022 4023	/* If there are blocks to remove, do it */4024	if (stop_block > first_block) {4025		ext4_lblk_t hole_len = stop_block - first_block;4026 4027		down_write(&EXT4_I(inode)->i_data_sem);4028		ext4_discard_preallocations(inode);4029 4030		ext4_es_remove_extent(inode, first_block, hole_len);4031 4032		if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))4033			ret = ext4_ext_remove_space(inode, first_block,4034						    stop_block - 1);4035		else4036			ret = ext4_ind_remove_space(handle, inode, first_block,4037						    stop_block);4038 4039		ext4_es_insert_extent(inode, first_block, hole_len, ~0,4040				      EXTENT_STATUS_HOLE, 0);4041		up_write(&EXT4_I(inode)->i_data_sem);4042	}4043	ext4_fc_track_range(handle, inode, first_block, stop_block);4044	if (IS_SYNC(inode))4045		ext4_handle_sync(handle);4046 4047	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));4048	ret2 = ext4_mark_inode_dirty(handle, inode);4049	if (unlikely(ret2))4050		ret = ret2;4051	if (ret >= 0)4052		ext4_update_inode_fsync_trans(handle, inode, 1);4053out_stop:4054	ext4_journal_stop(handle);4055out_dio:4056	filemap_invalidate_unlock(mapping);4057out_mutex:4058	inode_unlock(inode);4059	return ret;4060}4061 4062int ext4_inode_attach_jinode(struct inode *inode)4063{4064	struct ext4_inode_info *ei = EXT4_I(inode);4065	struct jbd2_inode *jinode;4066 4067	if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)4068		return 0;4069 4070	jinode = jbd2_alloc_inode(GFP_KERNEL);4071	spin_lock(&inode->i_lock);4072	if (!ei->jinode) {4073		if (!jinode) {4074			spin_unlock(&inode->i_lock);4075			return -ENOMEM;4076		}4077		ei->jinode = jinode;4078		jbd2_journal_init_jbd_inode(ei->jinode, inode);4079		jinode = NULL;4080	}4081	spin_unlock(&inode->i_lock);4082	if (unlikely(jinode != NULL))4083		jbd2_free_inode(jinode);4084	return 0;4085}4086 4087/*4088 * ext4_truncate()4089 *4090 * We block out ext4_get_block() block instantiations across the entire4091 * transaction, and VFS/VM ensures that ext4_truncate() cannot run4092 * simultaneously on behalf of the same inode.4093 *4094 * As we work through the truncate and commit bits of it to the journal there4095 * is one core, guiding principle: the file's tree must always be consistent on4096 * disk.  We must be able to restart the truncate after a crash.4097 *4098 * The file's tree may be transiently inconsistent in memory (although it4099 * probably isn't), but whenever we close off and commit a journal transaction,4100 * the contents of (the filesystem + the journal) must be consistent and4101 * restartable.  It's pretty simple, really: bottom up, right to left (although4102 * left-to-right works OK too).4103 *4104 * Note that at recovery time, journal replay occurs *before* the restart of4105 * truncate against the orphan inode list.4106 *4107 * The committed inode has the new, desired i_size (which is the same as4108 * i_disksize in this case).  After a crash, ext4_orphan_cleanup() will see4109 * that this inode's truncate did not complete and it will again call4110 * ext4_truncate() to have another go.  So there will be instantiated blocks4111 * to the right of the truncation point in a crashed ext4 filesystem.  But4112 * that's fine - as long as they are linked from the inode, the post-crash4113 * ext4_truncate() run will find them and release them.4114 */4115int ext4_truncate(struct inode *inode)4116{4117	struct ext4_inode_info *ei = EXT4_I(inode);4118	unsigned int credits;4119	int err = 0, err2;4120	handle_t *handle;4121	struct address_space *mapping = inode->i_mapping;4122 4123	/*4124	 * There is a possibility that we're either freeing the inode4125	 * or it's a completely new inode. In those cases we might not4126	 * have i_rwsem locked because it's not necessary.4127	 */4128	if (!(inode->i_state & (I_NEW|I_FREEING)))4129		WARN_ON(!inode_is_locked(inode));4130	trace_ext4_truncate_enter(inode);4131 4132	if (!ext4_can_truncate(inode))4133		goto out_trace;4134 4135	if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))4136		ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);4137 4138	if (ext4_has_inline_data(inode)) {4139		int has_inline = 1;4140 4141		err = ext4_inline_data_truncate(inode, &has_inline);4142		if (err || has_inline)4143			goto out_trace;4144	}4145 4146	/* If we zero-out tail of the page, we have to create jinode for jbd2 */4147	if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {4148		err = ext4_inode_attach_jinode(inode);4149		if (err)4150			goto out_trace;4151	}4152 4153	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))4154		credits = ext4_writepage_trans_blocks(inode);4155	else4156		credits = ext4_blocks_for_truncate(inode);4157 4158	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);4159	if (IS_ERR(handle)) {4160		err = PTR_ERR(handle);4161		goto out_trace;4162	}4163 4164	if (inode->i_size & (inode->i_sb->s_blocksize - 1))4165		ext4_block_truncate_page(handle, mapping, inode->i_size);4166 4167	/*4168	 * We add the inode to the orphan list, so that if this4169	 * truncate spans multiple transactions, and we crash, we will4170	 * resume the truncate when the filesystem recovers.  It also4171	 * marks the inode dirty, to catch the new size.4172	 *4173	 * Implication: the file must always be in a sane, consistent4174	 * truncatable state while each transaction commits.4175	 */4176	err = ext4_orphan_add(handle, inode);4177	if (err)4178		goto out_stop;4179 4180	down_write(&EXT4_I(inode)->i_data_sem);4181 4182	ext4_discard_preallocations(inode);4183 4184	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))4185		err = ext4_ext_truncate(handle, inode);4186	else4187		ext4_ind_truncate(handle, inode);4188 4189	up_write(&ei->i_data_sem);4190	if (err)4191		goto out_stop;4192 4193	if (IS_SYNC(inode))4194		ext4_handle_sync(handle);4195 4196out_stop:4197	/*4198	 * If this was a simple ftruncate() and the file will remain alive,4199	 * then we need to clear up the orphan record which we created above.4200	 * However, if this was a real unlink then we were called by4201	 * ext4_evict_inode(), and we allow that function to clean up the4202	 * orphan info for us.4203	 */4204	if (inode->i_nlink)4205		ext4_orphan_del(handle, inode);4206 4207	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));4208	err2 = ext4_mark_inode_dirty(handle, inode);4209	if (unlikely(err2 && !err))4210		err = err2;4211	ext4_journal_stop(handle);4212 4213out_trace:4214	trace_ext4_truncate_exit(inode);4215	return err;4216}4217 4218static inline u64 ext4_inode_peek_iversion(const struct inode *inode)4219{4220	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))4221		return inode_peek_iversion_raw(inode);4222	else4223		return inode_peek_iversion(inode);4224}4225 4226static int ext4_inode_blocks_set(struct ext4_inode *raw_inode,4227				 struct ext4_inode_info *ei)4228{4229	struct inode *inode = &(ei->vfs_inode);4230	u64 i_blocks = READ_ONCE(inode->i_blocks);4231	struct super_block *sb = inode->i_sb;4232 4233	if (i_blocks <= ~0U) {4234		/*4235		 * i_blocks can be represented in a 32 bit variable4236		 * as multiple of 512 bytes4237		 */4238		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);4239		raw_inode->i_blocks_high = 0;4240		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);4241		return 0;4242	}4243 4244	/*4245	 * This should never happen since sb->s_maxbytes should not have4246	 * allowed this, sb->s_maxbytes was set according to the huge_file4247	 * feature in ext4_fill_super().4248	 */4249	if (!ext4_has_feature_huge_file(sb))4250		return -EFSCORRUPTED;4251 4252	if (i_blocks <= 0xffffffffffffULL) {4253		/*4254		 * i_blocks can be represented in a 48 bit variable4255		 * as multiple of 512 bytes4256		 */4257		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);4258		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);4259		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);4260	} else {4261		ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);4262		/* i_block is stored in file system block size */4263		i_blocks = i_blocks >> (inode->i_blkbits - 9);4264		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);4265		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);4266	}4267	return 0;4268}4269 4270static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode)4271{4272	struct ext4_inode_info *ei = EXT4_I(inode);4273	uid_t i_uid;4274	gid_t i_gid;4275	projid_t i_projid;4276	int block;4277	int err;4278 4279	err = ext4_inode_blocks_set(raw_inode, ei);4280 4281	raw_inode->i_mode = cpu_to_le16(inode->i_mode);4282	i_uid = i_uid_read(inode);4283	i_gid = i_gid_read(inode);4284	i_projid = from_kprojid(&init_user_ns, ei->i_projid);4285	if (!(test_opt(inode->i_sb, NO_UID32))) {4286		raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));4287		raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));4288		/*4289		 * Fix up interoperability with old kernels. Otherwise,4290		 * old inodes get re-used with the upper 16 bits of the4291		 * uid/gid intact.4292		 */4293		if (ei->i_dtime && list_empty(&ei->i_orphan)) {4294			raw_inode->i_uid_high = 0;4295			raw_inode->i_gid_high = 0;4296		} else {4297			raw_inode->i_uid_high =4298				cpu_to_le16(high_16_bits(i_uid));4299			raw_inode->i_gid_high =4300				cpu_to_le16(high_16_bits(i_gid));4301		}4302	} else {4303		raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));4304		raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));4305		raw_inode->i_uid_high = 0;4306		raw_inode->i_gid_high = 0;4307	}4308	raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);4309 4310	EXT4_INODE_SET_CTIME(inode, raw_inode);4311	EXT4_INODE_SET_MTIME(inode, raw_inode);4312	EXT4_INODE_SET_ATIME(inode, raw_inode);4313	EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);4314 4315	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);4316	raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);4317	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))4318		raw_inode->i_file_acl_high =4319			cpu_to_le16(ei->i_file_acl >> 32);4320	raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);4321	ext4_isize_set(raw_inode, ei->i_disksize);4322 4323	raw_inode->i_generation = cpu_to_le32(inode->i_generation);4324	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {4325		if (old_valid_dev(inode->i_rdev)) {4326			raw_inode->i_block[0] =4327				cpu_to_le32(old_encode_dev(inode->i_rdev));4328			raw_inode->i_block[1] = 0;4329		} else {4330			raw_inode->i_block[0] = 0;4331			raw_inode->i_block[1] =4332				cpu_to_le32(new_encode_dev(inode->i_rdev));4333			raw_inode->i_block[2] = 0;4334		}4335	} else if (!ext4_has_inline_data(inode)) {4336		for (block = 0; block < EXT4_N_BLOCKS; block++)4337			raw_inode->i_block[block] = ei->i_data[block];4338	}4339 4340	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {4341		u64 ivers = ext4_inode_peek_iversion(inode);4342 4343		raw_inode->i_disk_version = cpu_to_le32(ivers);4344		if (ei->i_extra_isize) {4345			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))4346				raw_inode->i_version_hi =4347					cpu_to_le32(ivers >> 32);4348			raw_inode->i_extra_isize =4349				cpu_to_le16(ei->i_extra_isize);4350		}4351	}4352 4353	if (i_projid != EXT4_DEF_PROJID &&4354	    !ext4_has_feature_project(inode->i_sb))4355		err = err ?: -EFSCORRUPTED;4356 4357	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&4358	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))4359		raw_inode->i_projid = cpu_to_le32(i_projid);4360 4361	ext4_inode_csum_set(inode, raw_inode, ei);4362	return err;4363}4364 4365/*4366 * ext4_get_inode_loc returns with an extra refcount against the inode's4367 * underlying buffer_head on success. If we pass 'inode' and it does not4368 * have in-inode xattr, we have all inode data in memory that is needed4369 * to recreate the on-disk version of this inode.4370 */4371static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,4372				struct inode *inode, struct ext4_iloc *iloc,4373				ext4_fsblk_t *ret_block)4374{4375	struct ext4_group_desc	*gdp;4376	struct buffer_head	*bh;4377	ext4_fsblk_t		block;4378	struct blk_plug		plug;4379	int			inodes_per_block, inode_offset;4380 4381	iloc->bh = NULL;4382	if (ino < EXT4_ROOT_INO ||4383	    ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))4384		return -EFSCORRUPTED;4385 4386	iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);4387	gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);4388	if (!gdp)4389		return -EIO;4390 4391	/*4392	 * Figure out the offset within the block group inode table4393	 */4394	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;4395	inode_offset = ((ino - 1) %4396			EXT4_INODES_PER_GROUP(sb));4397	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);4398 4399	block = ext4_inode_table(sb, gdp);4400	if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) ||4401	    (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) {4402		ext4_error(sb, "Invalid inode table block %llu in "4403			   "block_group %u", block, iloc->block_group);4404		return -EFSCORRUPTED;4405	}4406	block += (inode_offset / inodes_per_block);4407 4408	bh = sb_getblk(sb, block);4409	if (unlikely(!bh))4410		return -ENOMEM;4411	if (ext4_buffer_uptodate(bh))4412		goto has_buffer;4413 4414	lock_buffer(bh);4415	if (ext4_buffer_uptodate(bh)) {4416		/* Someone brought it uptodate while we waited */4417		unlock_buffer(bh);4418		goto has_buffer;4419	}4420 4421	/*4422	 * If we have all information of the inode in memory and this4423	 * is the only valid inode in the block, we need not read the4424	 * block.4425	 */4426	if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {4427		struct buffer_head *bitmap_bh;4428		int i, start;4429 4430		start = inode_offset & ~(inodes_per_block - 1);4431 4432		/* Is the inode bitmap in cache? */4433		bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));4434		if (unlikely(!bitmap_bh))4435			goto make_io;4436 4437		/*4438		 * If the inode bitmap isn't in cache then the4439		 * optimisation may end up performing two reads instead4440		 * of one, so skip it.4441		 */4442		if (!buffer_uptodate(bitmap_bh)) {4443			brelse(bitmap_bh);4444			goto make_io;4445		}4446		for (i = start; i < start + inodes_per_block; i++) {4447			if (i == inode_offset)4448				continue;4449			if (ext4_test_bit(i, bitmap_bh->b_data))4450				break;4451		}4452		brelse(bitmap_bh);4453		if (i == start + inodes_per_block) {4454			struct ext4_inode *raw_inode =4455				(struct ext4_inode *) (bh->b_data + iloc->offset);4456 4457			/* all other inodes are free, so skip I/O */4458			memset(bh->b_data, 0, bh->b_size);4459			if (!ext4_test_inode_state(inode, EXT4_STATE_NEW))4460				ext4_fill_raw_inode(inode, raw_inode);4461			set_buffer_uptodate(bh);4462			unlock_buffer(bh);4463			goto has_buffer;4464		}4465	}4466 4467make_io:4468	/*4469	 * If we need to do any I/O, try to pre-readahead extra4470	 * blocks from the inode table.4471	 */4472	blk_start_plug(&plug);4473	if (EXT4_SB(sb)->s_inode_readahead_blks) {4474		ext4_fsblk_t b, end, table;4475		unsigned num;4476		__u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;4477 4478		table = ext4_inode_table(sb, gdp);4479		/* s_inode_readahead_blks is always a power of 2 */4480		b = block & ~((ext4_fsblk_t) ra_blks - 1);4481		if (table > b)4482			b = table;4483		end = b + ra_blks;4484		num = EXT4_INODES_PER_GROUP(sb);4485		if (ext4_has_group_desc_csum(sb))4486			num -= ext4_itable_unused_count(sb, gdp);4487		table += num / inodes_per_block;4488		if (end > table)4489			end = table;4490		while (b <= end)4491			ext4_sb_breadahead_unmovable(sb, b++);4492	}4493 4494	/*4495	 * There are other valid inodes in the buffer, this inode4496	 * has in-inode xattrs, or we don't have this inode in memory.4497	 * Read the block from disk.4498	 */4499	trace_ext4_load_inode(sb, ino);4500	ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL);4501	blk_finish_plug(&plug);4502	wait_on_buffer(bh);4503	ext4_simulate_fail_bh(sb, bh, EXT4_SIM_INODE_EIO);4504	if (!buffer_uptodate(bh)) {4505		if (ret_block)4506			*ret_block = block;4507		brelse(bh);4508		return -EIO;4509	}4510has_buffer:4511	iloc->bh = bh;4512	return 0;4513}4514 4515static int __ext4_get_inode_loc_noinmem(struct inode *inode,4516					struct ext4_iloc *iloc)4517{4518	ext4_fsblk_t err_blk = 0;4519	int ret;4520 4521	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc,4522					&err_blk);4523 4524	if (ret == -EIO)4525		ext4_error_inode_block(inode, err_blk, EIO,4526					"unable to read itable block");4527 4528	return ret;4529}4530 4531int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)4532{4533	ext4_fsblk_t err_blk = 0;4534	int ret;4535 4536	ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc,4537					&err_blk);4538 4539	if (ret == -EIO)4540		ext4_error_inode_block(inode, err_blk, EIO,4541					"unable to read itable block");4542 4543	return ret;4544}4545 4546 4547int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,4548			  struct ext4_iloc *iloc)4549{4550	return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL);4551}4552 4553static bool ext4_should_enable_dax(struct inode *inode)4554{4555	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);4556 4557	if (test_opt2(inode->i_sb, DAX_NEVER))4558		return false;4559	if (!S_ISREG(inode->i_mode))4560		return false;4561	if (ext4_should_journal_data(inode))4562		return false;4563	if (ext4_has_inline_data(inode))4564		return false;4565	if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))4566		return false;4567	if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))4568		return false;4569	if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))4570		return false;4571	if (test_opt(inode->i_sb, DAX_ALWAYS))4572		return true;4573 4574	return ext4_test_inode_flag(inode, EXT4_INODE_DAX);4575}4576 4577void ext4_set_inode_flags(struct inode *inode, bool init)4578{4579	unsigned int flags = EXT4_I(inode)->i_flags;4580	unsigned int new_fl = 0;4581 4582	WARN_ON_ONCE(IS_DAX(inode) && init);4583 4584	if (flags & EXT4_SYNC_FL)4585		new_fl |= S_SYNC;4586	if (flags & EXT4_APPEND_FL)4587		new_fl |= S_APPEND;4588	if (flags & EXT4_IMMUTABLE_FL)4589		new_fl |= S_IMMUTABLE;4590	if (flags & EXT4_NOATIME_FL)4591		new_fl |= S_NOATIME;4592	if (flags & EXT4_DIRSYNC_FL)4593		new_fl |= S_DIRSYNC;4594 4595	/* Because of the way inode_set_flags() works we must preserve S_DAX4596	 * here if already set. */4597	new_fl |= (inode->i_flags & S_DAX);4598	if (init && ext4_should_enable_dax(inode))4599		new_fl |= S_DAX;4600 4601	if (flags & EXT4_ENCRYPT_FL)4602		new_fl |= S_ENCRYPTED;4603	if (flags & EXT4_CASEFOLD_FL)4604		new_fl |= S_CASEFOLD;4605	if (flags & EXT4_VERITY_FL)4606		new_fl |= S_VERITY;4607	inode_set_flags(inode, new_fl,4608			S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|4609			S_ENCRYPTED|S_CASEFOLD|S_VERITY);4610}4611 4612static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,4613				  struct ext4_inode_info *ei)4614{4615	blkcnt_t i_blocks ;4616	struct inode *inode = &(ei->vfs_inode);4617	struct super_block *sb = inode->i_sb;4618 4619	if (ext4_has_feature_huge_file(sb)) {4620		/* we are using combined 48 bit field */4621		i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |4622					le32_to_cpu(raw_inode->i_blocks_lo);4623		if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {4624			/* i_blocks represent file system block size */4625			return i_blocks  << (inode->i_blkbits - 9);4626		} else {4627			return i_blocks;4628		}4629	} else {4630		return le32_to_cpu(raw_inode->i_blocks_lo);4631	}4632}4633 4634static inline int ext4_iget_extra_inode(struct inode *inode,4635					 struct ext4_inode *raw_inode,4636					 struct ext4_inode_info *ei)4637{4638	__le32 *magic = (void *)raw_inode +4639			EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;4640 4641	if (EXT4_INODE_HAS_XATTR_SPACE(inode)  &&4642	    *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {4643		int err;4644 4645		ext4_set_inode_state(inode, EXT4_STATE_XATTR);4646		err = ext4_find_inline_data_nolock(inode);4647		if (!err && ext4_has_inline_data(inode))4648			ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);4649		return err;4650	} else4651		EXT4_I(inode)->i_inline_off = 0;4652	return 0;4653}4654 4655int ext4_get_projid(struct inode *inode, kprojid_t *projid)4656{4657	if (!ext4_has_feature_project(inode->i_sb))4658		return -EOPNOTSUPP;4659	*projid = EXT4_I(inode)->i_projid;4660	return 0;4661}4662 4663/*4664 * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of4665 * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag4666 * set.4667 */4668static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)4669{4670	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))4671		inode_set_iversion_raw(inode, val);4672	else4673		inode_set_iversion_queried(inode, val);4674}4675 4676static const char *check_igot_inode(struct inode *inode, ext4_iget_flags flags)4677 4678{4679	if (flags & EXT4_IGET_EA_INODE) {4680		if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))4681			return "missing EA_INODE flag";4682		if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||4683		    EXT4_I(inode)->i_file_acl)4684			return "ea_inode with extended attributes";4685	} else {4686		if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))4687			return "unexpected EA_INODE flag";4688	}4689	if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD))4690		return "unexpected bad inode w/o EXT4_IGET_BAD";4691	return NULL;4692}4693 4694struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,4695			  ext4_iget_flags flags, const char *function,4696			  unsigned int line)4697{4698	struct ext4_iloc iloc;4699	struct ext4_inode *raw_inode;4700	struct ext4_inode_info *ei;4701	struct ext4_super_block *es = EXT4_SB(sb)->s_es;4702	struct inode *inode;4703	const char *err_str;4704	journal_t *journal = EXT4_SB(sb)->s_journal;4705	long ret;4706	loff_t size;4707	int block;4708	uid_t i_uid;4709	gid_t i_gid;4710	projid_t i_projid;4711 4712	if ((!(flags & EXT4_IGET_SPECIAL) &&4713	     ((ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) ||4714	      ino == le32_to_cpu(es->s_usr_quota_inum) ||4715	      ino == le32_to_cpu(es->s_grp_quota_inum) ||4716	      ino == le32_to_cpu(es->s_prj_quota_inum) ||4717	      ino == le32_to_cpu(es->s_orphan_file_inum))) ||4718	    (ino < EXT4_ROOT_INO) ||4719	    (ino > le32_to_cpu(es->s_inodes_count))) {4720		if (flags & EXT4_IGET_HANDLE)4721			return ERR_PTR(-ESTALE);4722		__ext4_error(sb, function, line, false, EFSCORRUPTED, 0,4723			     "inode #%lu: comm %s: iget: illegal inode #",4724			     ino, current->comm);4725		return ERR_PTR(-EFSCORRUPTED);4726	}4727 4728	inode = iget_locked(sb, ino);4729	if (!inode)4730		return ERR_PTR(-ENOMEM);4731	if (!(inode->i_state & I_NEW)) {4732		if ((err_str = check_igot_inode(inode, flags)) != NULL) {4733			ext4_error_inode(inode, function, line, 0, err_str);4734			iput(inode);4735			return ERR_PTR(-EFSCORRUPTED);4736		}4737		return inode;4738	}4739 4740	ei = EXT4_I(inode);4741	iloc.bh = NULL;4742 4743	ret = __ext4_get_inode_loc_noinmem(inode, &iloc);4744	if (ret < 0)4745		goto bad_inode;4746	raw_inode = ext4_raw_inode(&iloc);4747 4748	if ((flags & EXT4_IGET_HANDLE) &&4749	    (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {4750		ret = -ESTALE;4751		goto bad_inode;4752	}4753 4754	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {4755		ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);4756		if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >4757			EXT4_INODE_SIZE(inode->i_sb) ||4758		    (ei->i_extra_isize & 3)) {4759			ext4_error_inode(inode, function, line, 0,4760					 "iget: bad extra_isize %u "4761					 "(inode size %u)",4762					 ei->i_extra_isize,4763					 EXT4_INODE_SIZE(inode->i_sb));4764			ret = -EFSCORRUPTED;4765			goto bad_inode;4766		}4767	} else4768		ei->i_extra_isize = 0;4769 4770	/* Precompute checksum seed for inode metadata */4771	if (ext4_has_metadata_csum(sb)) {4772		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);4773		__u32 csum;4774		__le32 inum = cpu_to_le32(inode->i_ino);4775		__le32 gen = raw_inode->i_generation;4776		csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,4777				   sizeof(inum));4778		ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,4779					      sizeof(gen));4780	}4781 4782	if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||4783	    ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&4784	     (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {4785		ext4_error_inode_err(inode, function, line, 0,4786				EFSBADCRC, "iget: checksum invalid");4787		ret = -EFSBADCRC;4788		goto bad_inode;4789	}4790 4791	inode->i_mode = le16_to_cpu(raw_inode->i_mode);4792	i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);4793	i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);4794	if (ext4_has_feature_project(sb) &&4795	    EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&4796	    EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))4797		i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);4798	else4799		i_projid = EXT4_DEF_PROJID;4800 4801	if (!(test_opt(inode->i_sb, NO_UID32))) {4802		i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;4803		i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;4804	}4805	i_uid_write(inode, i_uid);4806	i_gid_write(inode, i_gid);4807	ei->i_projid = make_kprojid(&init_user_ns, i_projid);4808	set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));4809 4810	ext4_clear_state_flags(ei);	/* Only relevant on 32-bit archs */4811	ei->i_inline_off = 0;4812	ei->i_dir_start_lookup = 0;4813	ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);4814	/* We now have enough fields to check if the inode was active or not.4815	 * This is needed because nfsd might try to access dead inodes4816	 * the test is that same one that e2fsck uses4817	 * NeilBrown 1999oct154818	 */4819	if (inode->i_nlink == 0) {4820		if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL ||4821		     !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&4822		    ino != EXT4_BOOT_LOADER_INO) {4823			/* this inode is deleted or unallocated */4824			if (flags & EXT4_IGET_SPECIAL) {4825				ext4_error_inode(inode, function, line, 0,4826						 "iget: special inode unallocated");4827				ret = -EFSCORRUPTED;4828			} else4829				ret = -ESTALE;4830			goto bad_inode;4831		}4832		/* The only unlinked inodes we let through here have4833		 * valid i_mode and are being read by the orphan4834		 * recovery code: that's fine, we're about to complete4835		 * the process of deleting those.4836		 * OR it is the EXT4_BOOT_LOADER_INO which is4837		 * not initialized on a new filesystem. */4838	}4839	ei->i_flags = le32_to_cpu(raw_inode->i_flags);4840	ext4_set_inode_flags(inode, true);4841	inode->i_blocks = ext4_inode_blocks(raw_inode, ei);4842	ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);4843	if (ext4_has_feature_64bit(sb))4844		ei->i_file_acl |=4845			((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;4846	inode->i_size = ext4_isize(sb, raw_inode);4847	if ((size = i_size_read(inode)) < 0) {4848		ext4_error_inode(inode, function, line, 0,4849				 "iget: bad i_size value: %lld", size);4850		ret = -EFSCORRUPTED;4851		goto bad_inode;4852	}4853	/*4854	 * If dir_index is not enabled but there's dir with INDEX flag set,4855	 * we'd normally treat htree data as empty space. But with metadata4856	 * checksumming that corrupts checksums so forbid that.4857	 */4858	if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&4859	    ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {4860		ext4_error_inode(inode, function, line, 0,4861			 "iget: Dir with htree data on filesystem without dir_index feature.");4862		ret = -EFSCORRUPTED;4863		goto bad_inode;4864	}4865	ei->i_disksize = inode->i_size;4866#ifdef CONFIG_QUOTA4867	ei->i_reserved_quota = 0;4868#endif4869	inode->i_generation = le32_to_cpu(raw_inode->i_generation);4870	ei->i_block_group = iloc.block_group;4871	ei->i_last_alloc_group = ~0;4872	/*4873	 * NOTE! The in-memory inode i_data array is in little-endian order4874	 * even on big-endian machines: we do NOT byteswap the block numbers!4875	 */4876	for (block = 0; block < EXT4_N_BLOCKS; block++)4877		ei->i_data[block] = raw_inode->i_block[block];4878	INIT_LIST_HEAD(&ei->i_orphan);4879	ext4_fc_init_inode(&ei->vfs_inode);4880 4881	/*4882	 * Set transaction id's of transactions that have to be committed4883	 * to finish f[data]sync. We set them to currently running transaction4884	 * as we cannot be sure that the inode or some of its metadata isn't4885	 * part of the transaction - the inode could have been reclaimed and4886	 * now it is reread from disk.4887	 */4888	if (journal) {4889		transaction_t *transaction;4890		tid_t tid;4891 4892		read_lock(&journal->j_state_lock);4893		if (journal->j_running_transaction)4894			transaction = journal->j_running_transaction;4895		else4896			transaction = journal->j_committing_transaction;4897		if (transaction)4898			tid = transaction->t_tid;4899		else4900			tid = journal->j_commit_sequence;4901		read_unlock(&journal->j_state_lock);4902		ei->i_sync_tid = tid;4903		ei->i_datasync_tid = tid;4904	}4905 4906	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {4907		if (ei->i_extra_isize == 0) {4908			/* The extra space is currently unused. Use it. */4909			BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);4910			ei->i_extra_isize = sizeof(struct ext4_inode) -4911					    EXT4_GOOD_OLD_INODE_SIZE;4912		} else {4913			ret = ext4_iget_extra_inode(inode, raw_inode, ei);4914			if (ret)4915				goto bad_inode;4916		}4917	}4918 4919	EXT4_INODE_GET_CTIME(inode, raw_inode);4920	EXT4_INODE_GET_ATIME(inode, raw_inode);4921	EXT4_INODE_GET_MTIME(inode, raw_inode);4922	EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);4923 4924	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {4925		u64 ivers = le32_to_cpu(raw_inode->i_disk_version);4926 4927		if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {4928			if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))4929				ivers |=4930		    (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;4931		}4932		ext4_inode_set_iversion_queried(inode, ivers);4933	}4934 4935	ret = 0;4936	if (ei->i_file_acl &&4937	    !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {4938		ext4_error_inode(inode, function, line, 0,4939				 "iget: bad extended attribute block %llu",4940				 ei->i_file_acl);4941		ret = -EFSCORRUPTED;4942		goto bad_inode;4943	} else if (!ext4_has_inline_data(inode)) {4944		/* validate the block references in the inode */4945		if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&4946			(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||4947			(S_ISLNK(inode->i_mode) &&4948			!ext4_inode_is_fast_symlink(inode)))) {4949			if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))4950				ret = ext4_ext_check_inode(inode);4951			else4952				ret = ext4_ind_check_inode(inode);4953		}4954	}4955	if (ret)4956		goto bad_inode;4957 4958	if (S_ISREG(inode->i_mode)) {4959		inode->i_op = &ext4_file_inode_operations;4960		inode->i_fop = &ext4_file_operations;4961		ext4_set_aops(inode);4962	} else if (S_ISDIR(inode->i_mode)) {4963		inode->i_op = &ext4_dir_inode_operations;4964		inode->i_fop = &ext4_dir_operations;4965	} else if (S_ISLNK(inode->i_mode)) {4966		/* VFS does not allow setting these so must be corruption */4967		if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {4968			ext4_error_inode(inode, function, line, 0,4969					 "iget: immutable or append flags "4970					 "not allowed on symlinks");4971			ret = -EFSCORRUPTED;4972			goto bad_inode;4973		}4974		if (IS_ENCRYPTED(inode)) {4975			inode->i_op = &ext4_encrypted_symlink_inode_operations;4976		} else if (ext4_inode_is_fast_symlink(inode)) {4977			inode->i_link = (char *)ei->i_data;4978			inode->i_op = &ext4_fast_symlink_inode_operations;4979			nd_terminate_link(ei->i_data, inode->i_size,4980				sizeof(ei->i_data) - 1);4981		} else {4982			inode->i_op = &ext4_symlink_inode_operations;4983		}4984	} else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||4985	      S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {4986		inode->i_op = &ext4_special_inode_operations;4987		if (raw_inode->i_block[0])4988			init_special_inode(inode, inode->i_mode,4989			   old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));4990		else4991			init_special_inode(inode, inode->i_mode,4992			   new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));4993	} else if (ino == EXT4_BOOT_LOADER_INO) {4994		make_bad_inode(inode);4995	} else {4996		ret = -EFSCORRUPTED;4997		ext4_error_inode(inode, function, line, 0,4998				 "iget: bogus i_mode (%o)", inode->i_mode);4999		goto bad_inode;5000	}5001	if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) {5002		ext4_error_inode(inode, function, line, 0,5003				 "casefold flag without casefold feature");5004		ret = -EFSCORRUPTED;5005		goto bad_inode;5006	}5007	if ((err_str = check_igot_inode(inode, flags)) != NULL) {5008		ext4_error_inode(inode, function, line, 0, err_str);5009		ret = -EFSCORRUPTED;5010		goto bad_inode;5011	}5012 5013	brelse(iloc.bh);5014	unlock_new_inode(inode);5015	return inode;5016 5017bad_inode:5018	brelse(iloc.bh);5019	iget_failed(inode);5020	return ERR_PTR(ret);5021}5022 5023static void __ext4_update_other_inode_time(struct super_block *sb,5024					   unsigned long orig_ino,5025					   unsigned long ino,5026					   struct ext4_inode *raw_inode)5027{5028	struct inode *inode;5029 5030	inode = find_inode_by_ino_rcu(sb, ino);5031	if (!inode)5032		return;5033 5034	if (!inode_is_dirtytime_only(inode))5035		return;5036 5037	spin_lock(&inode->i_lock);5038	if (inode_is_dirtytime_only(inode)) {5039		struct ext4_inode_info	*ei = EXT4_I(inode);5040 5041		inode->i_state &= ~I_DIRTY_TIME;5042		spin_unlock(&inode->i_lock);5043 5044		spin_lock(&ei->i_raw_lock);5045		EXT4_INODE_SET_CTIME(inode, raw_inode);5046		EXT4_INODE_SET_MTIME(inode, raw_inode);5047		EXT4_INODE_SET_ATIME(inode, raw_inode);5048		ext4_inode_csum_set(inode, raw_inode, ei);5049		spin_unlock(&ei->i_raw_lock);5050		trace_ext4_other_inode_update_time(inode, orig_ino);5051		return;5052	}5053	spin_unlock(&inode->i_lock);5054}5055 5056/*5057 * Opportunistically update the other time fields for other inodes in5058 * the same inode table block.5059 */5060static void ext4_update_other_inodes_time(struct super_block *sb,5061					  unsigned long orig_ino, char *buf)5062{5063	unsigned long ino;5064	int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;5065	int inode_size = EXT4_INODE_SIZE(sb);5066 5067	/*5068	 * Calculate the first inode in the inode table block.  Inode5069	 * numbers are one-based.  That is, the first inode in a block5070	 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).5071	 */5072	ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;5073	rcu_read_lock();5074	for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {5075		if (ino == orig_ino)5076			continue;5077		__ext4_update_other_inode_time(sb, orig_ino, ino,5078					       (struct ext4_inode *)buf);5079	}5080	rcu_read_unlock();5081}5082 5083/*5084 * Post the struct inode info into an on-disk inode location in the5085 * buffer-cache.  This gobbles the caller's reference to the5086 * buffer_head in the inode location struct.5087 *5088 * The caller must have write access to iloc->bh.5089 */5090static int ext4_do_update_inode(handle_t *handle,5091				struct inode *inode,5092				struct ext4_iloc *iloc)5093{5094	struct ext4_inode *raw_inode = ext4_raw_inode(iloc);5095	struct ext4_inode_info *ei = EXT4_I(inode);5096	struct buffer_head *bh = iloc->bh;5097	struct super_block *sb = inode->i_sb;5098	int err;5099	int need_datasync = 0, set_large_file = 0;5100 5101	spin_lock(&ei->i_raw_lock);5102 5103	/*5104	 * For fields not tracked in the in-memory inode, initialise them5105	 * to zero for new inodes.5106	 */5107	if (ext4_test_inode_state(inode, EXT4_STATE_NEW))5108		memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);5109 5110	if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode))5111		need_datasync = 1;5112	if (ei->i_disksize > 0x7fffffffULL) {5113		if (!ext4_has_feature_large_file(sb) ||5114		    EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV))5115			set_large_file = 1;5116	}5117 5118	err = ext4_fill_raw_inode(inode, raw_inode);5119	spin_unlock(&ei->i_raw_lock);5120	if (err) {5121		EXT4_ERROR_INODE(inode, "corrupted inode contents");5122		goto out_brelse;5123	}5124 5125	if (inode->i_sb->s_flags & SB_LAZYTIME)5126		ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,5127					      bh->b_data);5128 5129	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");5130	err = ext4_handle_dirty_metadata(handle, NULL, bh);5131	if (err)5132		goto out_error;5133	ext4_clear_inode_state(inode, EXT4_STATE_NEW);5134	if (set_large_file) {5135		BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");5136		err = ext4_journal_get_write_access(handle, sb,5137						    EXT4_SB(sb)->s_sbh,5138						    EXT4_JTR_NONE);5139		if (err)5140			goto out_error;5141		lock_buffer(EXT4_SB(sb)->s_sbh);5142		ext4_set_feature_large_file(sb);5143		ext4_superblock_csum_set(sb);5144		unlock_buffer(EXT4_SB(sb)->s_sbh);5145		ext4_handle_sync(handle);5146		err = ext4_handle_dirty_metadata(handle, NULL,5147						 EXT4_SB(sb)->s_sbh);5148	}5149	ext4_update_inode_fsync_trans(handle, inode, need_datasync);5150out_error:5151	ext4_std_error(inode->i_sb, err);5152out_brelse:5153	brelse(bh);5154	return err;5155}5156 5157/*5158 * ext4_write_inode()5159 *5160 * We are called from a few places:5161 *5162 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.5163 *   Here, there will be no transaction running. We wait for any running5164 *   transaction to commit.5165 *5166 * - Within flush work (sys_sync(), kupdate and such).5167 *   We wait on commit, if told to.5168 *5169 * - Within iput_final() -> write_inode_now()5170 *   We wait on commit, if told to.5171 *5172 * In all cases it is actually safe for us to return without doing anything,5173 * because the inode has been copied into a raw inode buffer in5174 * ext4_mark_inode_dirty().  This is a correctness thing for WB_SYNC_ALL5175 * writeback.5176 *5177 * Note that we are absolutely dependent upon all inode dirtiers doing the5178 * right thing: they *must* call mark_inode_dirty() after dirtying info in5179 * which we are interested.5180 *5181 * It would be a bug for them to not do this.  The code:5182 *5183 *	mark_inode_dirty(inode)5184 *	stuff();5185 *	inode->i_size = expr;5186 *5187 * is in error because write_inode() could occur while `stuff()' is running,5188 * and the new i_size will be lost.  Plus the inode will no longer be on the5189 * superblock's dirty inode list.5190 */5191int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)5192{5193	int err;5194 5195	if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))5196		return 0;5197 5198	if (unlikely(ext4_forced_shutdown(inode->i_sb)))5199		return -EIO;5200 5201	if (EXT4_SB(inode->i_sb)->s_journal) {5202		if (ext4_journal_current_handle()) {5203			ext4_debug("called recursively, non-PF_MEMALLOC!\n");5204			dump_stack();5205			return -EIO;5206		}5207 5208		/*5209		 * No need to force transaction in WB_SYNC_NONE mode. Also5210		 * ext4_sync_fs() will force the commit after everything is5211		 * written.5212		 */5213		if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)5214			return 0;5215 5216		err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,5217						EXT4_I(inode)->i_sync_tid);5218	} else {5219		struct ext4_iloc iloc;5220 5221		err = __ext4_get_inode_loc_noinmem(inode, &iloc);5222		if (err)5223			return err;5224		/*5225		 * sync(2) will flush the whole buffer cache. No need to do5226		 * it here separately for each inode.5227		 */5228		if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)5229			sync_dirty_buffer(iloc.bh);5230		if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {5231			ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,5232					       "IO error syncing inode");5233			err = -EIO;5234		}5235		brelse(iloc.bh);5236	}5237	return err;5238}5239 5240/*5241 * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate5242 * buffers that are attached to a folio straddling i_size and are undergoing5243 * commit. In that case we have to wait for commit to finish and try again.5244 */5245static void ext4_wait_for_tail_page_commit(struct inode *inode)5246{5247	unsigned offset;5248	journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;5249	tid_t commit_tid;5250	int ret;5251	bool has_transaction;5252 5253	offset = inode->i_size & (PAGE_SIZE - 1);5254	/*5255	 * If the folio is fully truncated, we don't need to wait for any commit5256	 * (and we even should not as __ext4_journalled_invalidate_folio() may5257	 * strip all buffers from the folio but keep the folio dirty which can then5258	 * confuse e.g. concurrent ext4_writepages() seeing dirty folio without5259	 * buffers). Also we don't need to wait for any commit if all buffers in5260	 * the folio remain valid. This is most beneficial for the common case of5261	 * blocksize == PAGESIZE.5262	 */5263	if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))5264		return;5265	while (1) {5266		struct folio *folio = filemap_lock_folio(inode->i_mapping,5267				      inode->i_size >> PAGE_SHIFT);5268		if (IS_ERR(folio))5269			return;5270		ret = __ext4_journalled_invalidate_folio(folio, offset,5271						folio_size(folio) - offset);5272		folio_unlock(folio);5273		folio_put(folio);5274		if (ret != -EBUSY)5275			return;5276		has_transaction = false;5277		read_lock(&journal->j_state_lock);5278		if (journal->j_committing_transaction) {5279			commit_tid = journal->j_committing_transaction->t_tid;5280			has_transaction = true;5281		}5282		read_unlock(&journal->j_state_lock);5283		if (has_transaction)5284			jbd2_log_wait_commit(journal, commit_tid);5285	}5286}5287 5288/*5289 * ext4_setattr()5290 *5291 * Called from notify_change.5292 *5293 * We want to trap VFS attempts to truncate the file as soon as5294 * possible.  In particular, we want to make sure that when the VFS5295 * shrinks i_size, we put the inode on the orphan list and modify5296 * i_disksize immediately, so that during the subsequent flushing of5297 * dirty pages and freeing of disk blocks, we can guarantee that any5298 * commit will leave the blocks being flushed in an unused state on5299 * disk.  (On recovery, the inode will get truncated and the blocks will5300 * be freed, so we have a strong guarantee that no future commit will5301 * leave these blocks visible to the user.)5302 *5303 * Another thing we have to assure is that if we are in ordered mode5304 * and inode is still attached to the committing transaction, we must5305 * we start writeout of all the dirty pages which are being truncated.5306 * This way we are sure that all the data written in the previous5307 * transaction are already on disk (truncate waits for pages under5308 * writeback).5309 *5310 * Called with inode->i_rwsem down.5311 */5312int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry,5313		 struct iattr *attr)5314{5315	struct inode *inode = d_inode(dentry);5316	int error, rc = 0;5317	int orphan = 0;5318	const unsigned int ia_valid = attr->ia_valid;5319	bool inc_ivers = true;5320 5321	if (unlikely(ext4_forced_shutdown(inode->i_sb)))5322		return -EIO;5323 5324	if (unlikely(IS_IMMUTABLE(inode)))5325		return -EPERM;5326 5327	if (unlikely(IS_APPEND(inode) &&5328		     (ia_valid & (ATTR_MODE | ATTR_UID |5329				  ATTR_GID | ATTR_TIMES_SET))))5330		return -EPERM;5331 5332	error = setattr_prepare(idmap, dentry, attr);5333	if (error)5334		return error;5335 5336	error = fscrypt_prepare_setattr(dentry, attr);5337	if (error)5338		return error;5339 5340	error = fsverity_prepare_setattr(dentry, attr);5341	if (error)5342		return error;5343 5344	if (is_quota_modification(idmap, inode, attr)) {5345		error = dquot_initialize(inode);5346		if (error)5347			return error;5348	}5349 5350	if (i_uid_needs_update(idmap, attr, inode) ||5351	    i_gid_needs_update(idmap, attr, inode)) {5352		handle_t *handle;5353 5354		/* (user+group)*(old+new) structure, inode write (sb,5355		 * inode block, ? - but truncate inode update has it) */5356		handle = ext4_journal_start(inode, EXT4_HT_QUOTA,5357			(EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +5358			 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);5359		if (IS_ERR(handle)) {5360			error = PTR_ERR(handle);5361			goto err_out;5362		}5363 5364		/* dquot_transfer() calls back ext4_get_inode_usage() which5365		 * counts xattr inode references.5366		 */5367		down_read(&EXT4_I(inode)->xattr_sem);5368		error = dquot_transfer(idmap, inode, attr);5369		up_read(&EXT4_I(inode)->xattr_sem);5370 5371		if (error) {5372			ext4_journal_stop(handle);5373			return error;5374		}5375		/* Update corresponding info in inode so that everything is in5376		 * one transaction */5377		i_uid_update(idmap, attr, inode);5378		i_gid_update(idmap, attr, inode);5379		error = ext4_mark_inode_dirty(handle, inode);5380		ext4_journal_stop(handle);5381		if (unlikely(error)) {5382			return error;5383		}5384	}5385 5386	if (attr->ia_valid & ATTR_SIZE) {5387		handle_t *handle;5388		loff_t oldsize = inode->i_size;5389		loff_t old_disksize;5390		int shrink = (attr->ia_size < inode->i_size);5391 5392		if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {5393			struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);5394 5395			if (attr->ia_size > sbi->s_bitmap_maxbytes) {5396				return -EFBIG;5397			}5398		}5399		if (!S_ISREG(inode->i_mode)) {5400			return -EINVAL;5401		}5402 5403		if (attr->ia_size == inode->i_size)5404			inc_ivers = false;5405 5406		if (shrink) {5407			if (ext4_should_order_data(inode)) {5408				error = ext4_begin_ordered_truncate(inode,5409							    attr->ia_size);5410				if (error)5411					goto err_out;5412			}5413			/*5414			 * Blocks are going to be removed from the inode. Wait5415			 * for dio in flight.5416			 */5417			inode_dio_wait(inode);5418		}5419 5420		filemap_invalidate_lock(inode->i_mapping);5421 5422		rc = ext4_break_layouts(inode);5423		if (rc) {5424			filemap_invalidate_unlock(inode->i_mapping);5425			goto err_out;5426		}5427 5428		if (attr->ia_size != inode->i_size) {5429			handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);5430			if (IS_ERR(handle)) {5431				error = PTR_ERR(handle);5432				goto out_mmap_sem;5433			}5434			if (ext4_handle_valid(handle) && shrink) {5435				error = ext4_orphan_add(handle, inode);5436				orphan = 1;5437			}5438			/*5439			 * Update c/mtime on truncate up, ext4_truncate() will5440			 * update c/mtime in shrink case below5441			 */5442			if (!shrink)5443				inode_set_mtime_to_ts(inode,5444						      inode_set_ctime_current(inode));5445 5446			if (shrink)5447				ext4_fc_track_range(handle, inode,5448					(attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>5449					inode->i_sb->s_blocksize_bits,5450					EXT_MAX_BLOCKS - 1);5451			else5452				ext4_fc_track_range(5453					handle, inode,5454					(oldsize > 0 ? oldsize - 1 : oldsize) >>5455					inode->i_sb->s_blocksize_bits,5456					(attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>5457					inode->i_sb->s_blocksize_bits);5458 5459			down_write(&EXT4_I(inode)->i_data_sem);5460			old_disksize = EXT4_I(inode)->i_disksize;5461			EXT4_I(inode)->i_disksize = attr->ia_size;5462			rc = ext4_mark_inode_dirty(handle, inode);5463			if (!error)5464				error = rc;5465			/*5466			 * We have to update i_size under i_data_sem together5467			 * with i_disksize to avoid races with writeback code5468			 * running ext4_wb_update_i_disksize().5469			 */5470			if (!error)5471				i_size_write(inode, attr->ia_size);5472			else5473				EXT4_I(inode)->i_disksize = old_disksize;5474			up_write(&EXT4_I(inode)->i_data_sem);5475			ext4_journal_stop(handle);5476			if (error)5477				goto out_mmap_sem;5478			if (!shrink) {5479				pagecache_isize_extended(inode, oldsize,5480							 inode->i_size);5481			} else if (ext4_should_journal_data(inode)) {5482				ext4_wait_for_tail_page_commit(inode);5483			}5484		}5485 5486		/*5487		 * Truncate pagecache after we've waited for commit5488		 * in data=journal mode to make pages freeable.5489		 */5490		truncate_pagecache(inode, inode->i_size);5491		/*5492		 * Call ext4_truncate() even if i_size didn't change to5493		 * truncate possible preallocated blocks.5494		 */5495		if (attr->ia_size <= oldsize) {5496			rc = ext4_truncate(inode);5497			if (rc)5498				error = rc;5499		}5500out_mmap_sem:5501		filemap_invalidate_unlock(inode->i_mapping);5502	}5503 5504	if (!error) {5505		if (inc_ivers)5506			inode_inc_iversion(inode);5507		setattr_copy(idmap, inode, attr);5508		mark_inode_dirty(inode);5509	}5510 5511	/*5512	 * If the call to ext4_truncate failed to get a transaction handle at5513	 * all, we need to clean up the in-core orphan list manually.5514	 */5515	if (orphan && inode->i_nlink)5516		ext4_orphan_del(NULL, inode);5517 5518	if (!error && (ia_valid & ATTR_MODE))5519		rc = posix_acl_chmod(idmap, dentry, inode->i_mode);5520 5521err_out:5522	if  (error)5523		ext4_std_error(inode->i_sb, error);5524	if (!error)5525		error = rc;5526	return error;5527}5528 5529u32 ext4_dio_alignment(struct inode *inode)5530{5531	if (fsverity_active(inode))5532		return 0;5533	if (ext4_should_journal_data(inode))5534		return 0;5535	if (ext4_has_inline_data(inode))5536		return 0;5537	if (IS_ENCRYPTED(inode)) {5538		if (!fscrypt_dio_supported(inode))5539			return 0;5540		return i_blocksize(inode);5541	}5542	return 1; /* use the iomap defaults */5543}5544 5545int ext4_getattr(struct mnt_idmap *idmap, const struct path *path,5546		 struct kstat *stat, u32 request_mask, unsigned int query_flags)5547{5548	struct inode *inode = d_inode(path->dentry);5549	struct ext4_inode *raw_inode;5550	struct ext4_inode_info *ei = EXT4_I(inode);5551	unsigned int flags;5552 5553	if ((request_mask & STATX_BTIME) &&5554	    EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {5555		stat->result_mask |= STATX_BTIME;5556		stat->btime.tv_sec = ei->i_crtime.tv_sec;5557		stat->btime.tv_nsec = ei->i_crtime.tv_nsec;5558	}5559 5560	/*5561	 * Return the DIO alignment restrictions if requested.  We only return5562	 * this information when requested, since on encrypted files it might5563	 * take a fair bit of work to get if the file wasn't opened recently.5564	 */5565	if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {5566		u32 dio_align = ext4_dio_alignment(inode);5567 5568		stat->result_mask |= STATX_DIOALIGN;5569		if (dio_align == 1) {5570			struct block_device *bdev = inode->i_sb->s_bdev;5571 5572			/* iomap defaults */5573			stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;5574			stat->dio_offset_align = bdev_logical_block_size(bdev);5575		} else {5576			stat->dio_mem_align = dio_align;5577			stat->dio_offset_align = dio_align;5578		}5579	}5580 5581	flags = ei->i_flags & EXT4_FL_USER_VISIBLE;5582	if (flags & EXT4_APPEND_FL)5583		stat->attributes |= STATX_ATTR_APPEND;5584	if (flags & EXT4_COMPR_FL)5585		stat->attributes |= STATX_ATTR_COMPRESSED;5586	if (flags & EXT4_ENCRYPT_FL)5587		stat->attributes |= STATX_ATTR_ENCRYPTED;5588	if (flags & EXT4_IMMUTABLE_FL)5589		stat->attributes |= STATX_ATTR_IMMUTABLE;5590	if (flags & EXT4_NODUMP_FL)5591		stat->attributes |= STATX_ATTR_NODUMP;5592	if (flags & EXT4_VERITY_FL)5593		stat->attributes |= STATX_ATTR_VERITY;5594 5595	stat->attributes_mask |= (STATX_ATTR_APPEND |5596				  STATX_ATTR_COMPRESSED |5597				  STATX_ATTR_ENCRYPTED |5598				  STATX_ATTR_IMMUTABLE |5599				  STATX_ATTR_NODUMP |5600				  STATX_ATTR_VERITY);5601 5602	generic_fillattr(idmap, request_mask, inode, stat);5603	return 0;5604}5605 5606int ext4_file_getattr(struct mnt_idmap *idmap,5607		      const struct path *path, struct kstat *stat,5608		      u32 request_mask, unsigned int query_flags)5609{5610	struct inode *inode = d_inode(path->dentry);5611	u64 delalloc_blocks;5612 5613	ext4_getattr(idmap, path, stat, request_mask, query_flags);5614 5615	/*5616	 * If there is inline data in the inode, the inode will normally not5617	 * have data blocks allocated (it may have an external xattr block).5618	 * Report at least one sector for such files, so tools like tar, rsync,5619	 * others don't incorrectly think the file is completely sparse.5620	 */5621	if (unlikely(ext4_has_inline_data(inode)))5622		stat->blocks += (stat->size + 511) >> 9;5623 5624	/*5625	 * We can't update i_blocks if the block allocation is delayed5626	 * otherwise in the case of system crash before the real block5627	 * allocation is done, we will have i_blocks inconsistent with5628	 * on-disk file blocks.5629	 * We always keep i_blocks updated together with real5630	 * allocation. But to not confuse with user, stat5631	 * will return the blocks that include the delayed allocation5632	 * blocks for this file.5633	 */5634	delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),5635				   EXT4_I(inode)->i_reserved_data_blocks);5636	stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);5637	return 0;5638}5639 5640static int ext4_index_trans_blocks(struct inode *inode, int lblocks,5641				   int pextents)5642{5643	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))5644		return ext4_ind_trans_blocks(inode, lblocks);5645	return ext4_ext_index_trans_blocks(inode, pextents);5646}5647 5648/*5649 * Account for index blocks, block groups bitmaps and block group5650 * descriptor blocks if modify datablocks and index blocks5651 * worse case, the indexs blocks spread over different block groups5652 *5653 * If datablocks are discontiguous, they are possible to spread over5654 * different block groups too. If they are contiguous, with flexbg,5655 * they could still across block group boundary.5656 *5657 * Also account for superblock, inode, quota and xattr blocks5658 */5659static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,5660				  int pextents)5661{5662	ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);5663	int gdpblocks;5664	int idxblocks;5665	int ret;5666 5667	/*5668	 * How many index blocks need to touch to map @lblocks logical blocks5669	 * to @pextents physical extents?5670	 */5671	idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);5672 5673	ret = idxblocks;5674 5675	/*5676	 * Now let's see how many group bitmaps and group descriptors need5677	 * to account5678	 */5679	groups = idxblocks + pextents;5680	gdpblocks = groups;5681	if (groups > ngroups)5682		groups = ngroups;5683	if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)5684		gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;5685 5686	/* bitmaps and block group descriptor blocks */5687	ret += groups + gdpblocks;5688 5689	/* Blocks for super block, inode, quota and xattr blocks */5690	ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);5691 5692	return ret;5693}5694 5695/*5696 * Calculate the total number of credits to reserve to fit5697 * the modification of a single pages into a single transaction,5698 * which may include multiple chunks of block allocations.5699 *5700 * This could be called via ext4_write_begin()5701 *5702 * We need to consider the worse case, when5703 * one new block per extent.5704 */5705int ext4_writepage_trans_blocks(struct inode *inode)5706{5707	int bpp = ext4_journal_blocks_per_page(inode);5708	int ret;5709 5710	ret = ext4_meta_trans_blocks(inode, bpp, bpp);5711 5712	/* Account for data blocks for journalled mode */5713	if (ext4_should_journal_data(inode))5714		ret += bpp;5715	return ret;5716}5717 5718/*5719 * Calculate the journal credits for a chunk of data modification.5720 *5721 * This is called from DIO, fallocate or whoever calling5722 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.5723 *5724 * journal buffers for data blocks are not included here, as DIO5725 * and fallocate do no need to journal data buffers.5726 */5727int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)5728{5729	return ext4_meta_trans_blocks(inode, nrblocks, 1);5730}5731 5732/*5733 * The caller must have previously called ext4_reserve_inode_write().5734 * Give this, we know that the caller already has write access to iloc->bh.5735 */5736int ext4_mark_iloc_dirty(handle_t *handle,5737			 struct inode *inode, struct ext4_iloc *iloc)5738{5739	int err = 0;5740 5741	if (unlikely(ext4_forced_shutdown(inode->i_sb))) {5742		put_bh(iloc->bh);5743		return -EIO;5744	}5745	ext4_fc_track_inode(handle, inode);5746 5747	/* the do_update_inode consumes one bh->b_count */5748	get_bh(iloc->bh);5749 5750	/* ext4_do_update_inode() does jbd2_journal_dirty_metadata */5751	err = ext4_do_update_inode(handle, inode, iloc);5752	put_bh(iloc->bh);5753	return err;5754}5755 5756/*5757 * On success, We end up with an outstanding reference count against5758 * iloc->bh.  This _must_ be cleaned up later.5759 */5760 5761int5762ext4_reserve_inode_write(handle_t *handle, struct inode *inode,5763			 struct ext4_iloc *iloc)5764{5765	int err;5766 5767	if (unlikely(ext4_forced_shutdown(inode->i_sb)))5768		return -EIO;5769 5770	err = ext4_get_inode_loc(inode, iloc);5771	if (!err) {5772		BUFFER_TRACE(iloc->bh, "get_write_access");5773		err = ext4_journal_get_write_access(handle, inode->i_sb,5774						    iloc->bh, EXT4_JTR_NONE);5775		if (err) {5776			brelse(iloc->bh);5777			iloc->bh = NULL;5778		}5779	}5780	ext4_std_error(inode->i_sb, err);5781	return err;5782}5783 5784static int __ext4_expand_extra_isize(struct inode *inode,5785				     unsigned int new_extra_isize,5786				     struct ext4_iloc *iloc,5787				     handle_t *handle, int *no_expand)5788{5789	struct ext4_inode *raw_inode;5790	struct ext4_xattr_ibody_header *header;5791	unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);5792	struct ext4_inode_info *ei = EXT4_I(inode);5793	int error;5794 5795	/* this was checked at iget time, but double check for good measure */5796	if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||5797	    (ei->i_extra_isize & 3)) {5798		EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",5799				 ei->i_extra_isize,5800				 EXT4_INODE_SIZE(inode->i_sb));5801		return -EFSCORRUPTED;5802	}5803	if ((new_extra_isize < ei->i_extra_isize) ||5804	    (new_extra_isize < 4) ||5805	    (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))5806		return -EINVAL;	/* Should never happen */5807 5808	raw_inode = ext4_raw_inode(iloc);5809 5810	header = IHDR(inode, raw_inode);5811 5812	/* No extended attributes present */5813	if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||5814	    header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {5815		memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +5816		       EXT4_I(inode)->i_extra_isize, 0,5817		       new_extra_isize - EXT4_I(inode)->i_extra_isize);5818		EXT4_I(inode)->i_extra_isize = new_extra_isize;5819		return 0;5820	}5821 5822	/*5823	 * We may need to allocate external xattr block so we need quotas5824	 * initialized. Here we can be called with various locks held so we5825	 * cannot affort to initialize quotas ourselves. So just bail.5826	 */5827	if (dquot_initialize_needed(inode))5828		return -EAGAIN;5829 5830	/* try to expand with EAs present */5831	error = ext4_expand_extra_isize_ea(inode, new_extra_isize,5832					   raw_inode, handle);5833	if (error) {5834		/*5835		 * Inode size expansion failed; don't try again5836		 */5837		*no_expand = 1;5838	}5839 5840	return error;5841}5842 5843/*5844 * Expand an inode by new_extra_isize bytes.5845 * Returns 0 on success or negative error number on failure.5846 */5847static int ext4_try_to_expand_extra_isize(struct inode *inode,5848					  unsigned int new_extra_isize,5849					  struct ext4_iloc iloc,5850					  handle_t *handle)5851{5852	int no_expand;5853	int error;5854 5855	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))5856		return -EOVERFLOW;5857 5858	/*5859	 * In nojournal mode, we can immediately attempt to expand5860	 * the inode.  When journaled, we first need to obtain extra5861	 * buffer credits since we may write into the EA block5862	 * with this same handle. If journal_extend fails, then it will5863	 * only result in a minor loss of functionality for that inode.5864	 * If this is felt to be critical, then e2fsck should be run to5865	 * force a large enough s_min_extra_isize.5866	 */5867	if (ext4_journal_extend(handle,5868				EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)5869		return -ENOSPC;5870 5871	if (ext4_write_trylock_xattr(inode, &no_expand) == 0)5872		return -EBUSY;5873 5874	error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,5875					  handle, &no_expand);5876	ext4_write_unlock_xattr(inode, &no_expand);5877 5878	return error;5879}5880 5881int ext4_expand_extra_isize(struct inode *inode,5882			    unsigned int new_extra_isize,5883			    struct ext4_iloc *iloc)5884{5885	handle_t *handle;5886	int no_expand;5887	int error, rc;5888 5889	if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {5890		brelse(iloc->bh);5891		return -EOVERFLOW;5892	}5893 5894	handle = ext4_journal_start(inode, EXT4_HT_INODE,5895				    EXT4_DATA_TRANS_BLOCKS(inode->i_sb));5896	if (IS_ERR(handle)) {5897		error = PTR_ERR(handle);5898		brelse(iloc->bh);5899		return error;5900	}5901 5902	ext4_write_lock_xattr(inode, &no_expand);5903 5904	BUFFER_TRACE(iloc->bh, "get_write_access");5905	error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,5906					      EXT4_JTR_NONE);5907	if (error) {5908		brelse(iloc->bh);5909		goto out_unlock;5910	}5911 5912	error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,5913					  handle, &no_expand);5914 5915	rc = ext4_mark_iloc_dirty(handle, inode, iloc);5916	if (!error)5917		error = rc;5918 5919out_unlock:5920	ext4_write_unlock_xattr(inode, &no_expand);5921	ext4_journal_stop(handle);5922	return error;5923}5924 5925/*5926 * What we do here is to mark the in-core inode as clean with respect to inode5927 * dirtiness (it may still be data-dirty).5928 * This means that the in-core inode may be reaped by prune_icache5929 * without having to perform any I/O.  This is a very good thing,5930 * because *any* task may call prune_icache - even ones which5931 * have a transaction open against a different journal.5932 *5933 * Is this cheating?  Not really.  Sure, we haven't written the5934 * inode out, but prune_icache isn't a user-visible syncing function.5935 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)5936 * we start and wait on commits.5937 */5938int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,5939				const char *func, unsigned int line)5940{5941	struct ext4_iloc iloc;5942	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);5943	int err;5944 5945	might_sleep();5946	trace_ext4_mark_inode_dirty(inode, _RET_IP_);5947	err = ext4_reserve_inode_write(handle, inode, &iloc);5948	if (err)5949		goto out;5950 5951	if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)5952		ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,5953					       iloc, handle);5954 5955	err = ext4_mark_iloc_dirty(handle, inode, &iloc);5956out:5957	if (unlikely(err))5958		ext4_error_inode_err(inode, func, line, 0, err,5959					"mark_inode_dirty error");5960	return err;5961}5962 5963/*5964 * ext4_dirty_inode() is called from __mark_inode_dirty()5965 *5966 * We're really interested in the case where a file is being extended.5967 * i_size has been changed by generic_commit_write() and we thus need5968 * to include the updated inode in the current transaction.5969 *5970 * Also, dquot_alloc_block() will always dirty the inode when blocks5971 * are allocated to the file.5972 *5973 * If the inode is marked synchronous, we don't honour that here - doing5974 * so would cause a commit on atime updates, which we don't bother doing.5975 * We handle synchronous inodes at the highest possible level.5976 */5977void ext4_dirty_inode(struct inode *inode, int flags)5978{5979	handle_t *handle;5980 5981	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);5982	if (IS_ERR(handle))5983		return;5984	ext4_mark_inode_dirty(handle, inode);5985	ext4_journal_stop(handle);5986}5987 5988int ext4_change_inode_journal_flag(struct inode *inode, int val)5989{5990	journal_t *journal;5991	handle_t *handle;5992	int err;5993	int alloc_ctx;5994 5995	/*5996	 * We have to be very careful here: changing a data block's5997	 * journaling status dynamically is dangerous.  If we write a5998	 * data block to the journal, change the status and then delete5999	 * that block, we risk forgetting to revoke the old log record6000	 * from the journal and so a subsequent replay can corrupt data.6001	 * So, first we make sure that the journal is empty and that6002	 * nobody is changing anything.6003	 */6004 6005	journal = EXT4_JOURNAL(inode);6006	if (!journal)6007		return 0;6008	if (is_journal_aborted(journal))6009		return -EROFS;6010 6011	/* Wait for all existing dio workers */6012	inode_dio_wait(inode);6013 6014	/*6015	 * Before flushing the journal and switching inode's aops, we have6016	 * to flush all dirty data the inode has. There can be outstanding6017	 * delayed allocations, there can be unwritten extents created by6018	 * fallocate or buffered writes in dioread_nolock mode covered by6019	 * dirty data which can be converted only after flushing the dirty6020	 * data (and journalled aops don't know how to handle these cases).6021	 */6022	if (val) {6023		filemap_invalidate_lock(inode->i_mapping);6024		err = filemap_write_and_wait(inode->i_mapping);6025		if (err < 0) {6026			filemap_invalidate_unlock(inode->i_mapping);6027			return err;6028		}6029	}6030 6031	alloc_ctx = ext4_writepages_down_write(inode->i_sb);6032	jbd2_journal_lock_updates(journal);6033 6034	/*6035	 * OK, there are no updates running now, and all cached data is6036	 * synced to disk.  We are now in a completely consistent state6037	 * which doesn't have anything in the journal, and we know that6038	 * no filesystem updates are running, so it is safe to modify6039	 * the inode's in-core data-journaling state flag now.6040	 */6041 6042	if (val)6043		ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);6044	else {6045		err = jbd2_journal_flush(journal, 0);6046		if (err < 0) {6047			jbd2_journal_unlock_updates(journal);6048			ext4_writepages_up_write(inode->i_sb, alloc_ctx);6049			return err;6050		}6051		ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);6052	}6053	ext4_set_aops(inode);6054 6055	jbd2_journal_unlock_updates(journal);6056	ext4_writepages_up_write(inode->i_sb, alloc_ctx);6057 6058	if (val)6059		filemap_invalidate_unlock(inode->i_mapping);6060 6061	/* Finally we can mark the inode as dirty. */6062 6063	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);6064	if (IS_ERR(handle))6065		return PTR_ERR(handle);6066 6067	ext4_fc_mark_ineligible(inode->i_sb,6068		EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);6069	err = ext4_mark_inode_dirty(handle, inode);6070	ext4_handle_sync(handle);6071	ext4_journal_stop(handle);6072	ext4_std_error(inode->i_sb, err);6073 6074	return err;6075}6076 6077static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,6078			    struct buffer_head *bh)6079{6080	return !buffer_mapped(bh);6081}6082 6083vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)6084{6085	struct vm_area_struct *vma = vmf->vma;6086	struct folio *folio = page_folio(vmf->page);6087	loff_t size;6088	unsigned long len;6089	int err;6090	vm_fault_t ret;6091	struct file *file = vma->vm_file;6092	struct inode *inode = file_inode(file);6093	struct address_space *mapping = inode->i_mapping;6094	handle_t *handle;6095	get_block_t *get_block;6096	int retries = 0;6097 6098	if (unlikely(IS_IMMUTABLE(inode)))6099		return VM_FAULT_SIGBUS;6100 6101	sb_start_pagefault(inode->i_sb);6102	file_update_time(vma->vm_file);6103 6104	filemap_invalidate_lock_shared(mapping);6105 6106	err = ext4_convert_inline_data(inode);6107	if (err)6108		goto out_ret;6109 6110	/*6111	 * On data journalling we skip straight to the transaction handle:6112	 * there's no delalloc; page truncated will be checked later; the6113	 * early return w/ all buffers mapped (calculates size/len) can't6114	 * be used; and there's no dioread_nolock, so only ext4_get_block.6115	 */6116	if (ext4_should_journal_data(inode))6117		goto retry_alloc;6118 6119	/* Delalloc case is easy... */6120	if (test_opt(inode->i_sb, DELALLOC) &&6121	    !ext4_nonda_switch(inode->i_sb)) {6122		do {6123			err = block_page_mkwrite(vma, vmf,6124						   ext4_da_get_block_prep);6125		} while (err == -ENOSPC &&6126		       ext4_should_retry_alloc(inode->i_sb, &retries));6127		goto out_ret;6128	}6129 6130	folio_lock(folio);6131	size = i_size_read(inode);6132	/* Page got truncated from under us? */6133	if (folio->mapping != mapping || folio_pos(folio) > size) {6134		folio_unlock(folio);6135		ret = VM_FAULT_NOPAGE;6136		goto out;6137	}6138 6139	len = folio_size(folio);6140	if (folio_pos(folio) + len > size)6141		len = size - folio_pos(folio);6142	/*6143	 * Return if we have all the buffers mapped. This avoids the need to do6144	 * journal_start/journal_stop which can block and take a long time6145	 *6146	 * This cannot be done for data journalling, as we have to add the6147	 * inode to the transaction's list to writeprotect pages on commit.6148	 */6149	if (folio_buffers(folio)) {6150		if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio),6151					    0, len, NULL,6152					    ext4_bh_unmapped)) {6153			/* Wait so that we don't change page under IO */6154			folio_wait_stable(folio);6155			ret = VM_FAULT_LOCKED;6156			goto out;6157		}6158	}6159	folio_unlock(folio);6160	/* OK, we need to fill the hole... */6161	if (ext4_should_dioread_nolock(inode))6162		get_block = ext4_get_block_unwritten;6163	else6164		get_block = ext4_get_block;6165retry_alloc:6166	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,6167				    ext4_writepage_trans_blocks(inode));6168	if (IS_ERR(handle)) {6169		ret = VM_FAULT_SIGBUS;6170		goto out;6171	}6172	/*6173	 * Data journalling can't use block_page_mkwrite() because it6174	 * will set_buffer_dirty() before do_journal_get_write_access()6175	 * thus might hit warning messages for dirty metadata buffers.6176	 */6177	if (!ext4_should_journal_data(inode)) {6178		err = block_page_mkwrite(vma, vmf, get_block);6179	} else {6180		folio_lock(folio);6181		size = i_size_read(inode);6182		/* Page got truncated from under us? */6183		if (folio->mapping != mapping || folio_pos(folio) > size) {6184			ret = VM_FAULT_NOPAGE;6185			goto out_error;6186		}6187 6188		len = folio_size(folio);6189		if (folio_pos(folio) + len > size)6190			len = size - folio_pos(folio);6191 6192		err = ext4_block_write_begin(handle, folio, 0, len,6193					     ext4_get_block);6194		if (!err) {6195			ret = VM_FAULT_SIGBUS;6196			if (ext4_journal_folio_buffers(handle, folio, len))6197				goto out_error;6198		} else {6199			folio_unlock(folio);6200		}6201	}6202	ext4_journal_stop(handle);6203	if (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))6204		goto retry_alloc;6205out_ret:6206	ret = vmf_fs_error(err);6207out:6208	filemap_invalidate_unlock_shared(mapping);6209	sb_end_pagefault(inode->i_sb);6210	return ret;6211out_error:6212	folio_unlock(folio);6213	ext4_journal_stop(handle);6214	goto out;6215}6216