533 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/*3 * Copyright (C) 2017-2018 HUAWEI, Inc.4 * https://www.huawei.com/5 * Copyright (C) 2021, Alibaba Cloud6 */7#ifndef __EROFS_INTERNAL_H8#define __EROFS_INTERNAL_H9 10#include <linux/fs.h>11#include <linux/dax.h>12#include <linux/dcache.h>13#include <linux/mm.h>14#include <linux/module.h>15#include <linux/pagemap.h>16#include <linux/bio.h>17#include <linux/magic.h>18#include <linux/slab.h>19#include <linux/vmalloc.h>20#include <linux/iomap.h>21#include "erofs_fs.h"22 23/* redefine pr_fmt "erofs: " */24#undef pr_fmt25#define pr_fmt(fmt) "erofs: " fmt26 27__printf(3, 4) void _erofs_err(struct super_block *sb,28 const char *function, const char *fmt, ...);29#define erofs_err(sb, fmt, ...) \30 _erofs_err(sb, __func__, fmt "\n", ##__VA_ARGS__)31__printf(3, 4) void _erofs_info(struct super_block *sb,32 const char *function, const char *fmt, ...);33#define erofs_info(sb, fmt, ...) \34 _erofs_info(sb, __func__, fmt "\n", ##__VA_ARGS__)35#ifdef CONFIG_EROFS_FS_DEBUG36#define DBG_BUGON BUG_ON37#else38#define DBG_BUGON(x) ((void)(x))39#endif /* !CONFIG_EROFS_FS_DEBUG */40 41/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */42#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V143 44typedef u64 erofs_nid_t;45typedef u64 erofs_off_t;46/* data type for filesystem-wide blocks number */47typedef u32 erofs_blk_t;48 49struct erofs_device_info {50 char *path;51 struct erofs_fscache *fscache;52 struct file *file;53 struct dax_device *dax_dev;54 u64 dax_part_off;55 56 u32 blocks;57 u32 mapped_blkaddr;58};59 60enum {61 EROFS_SYNC_DECOMPRESS_AUTO,62 EROFS_SYNC_DECOMPRESS_FORCE_ON,63 EROFS_SYNC_DECOMPRESS_FORCE_OFF64};65 66struct erofs_mount_opts {67 /* current strategy of how to use managed cache */68 unsigned char cache_strategy;69 /* strategy of sync decompression (0 - auto, 1 - force on, 2 - force off) */70 unsigned int sync_decompress;71 /* threshold for decompression synchronously */72 unsigned int max_sync_decompress_pages;73 unsigned int mount_opt;74};75 76struct erofs_dev_context {77 struct idr tree;78 struct rw_semaphore rwsem;79 80 unsigned int extra_devices;81 bool flatdev;82};83 84/* all filesystem-wide lz4 configurations */85struct erofs_sb_lz4_info {86 /* # of pages needed for EROFS lz4 rolling decompression */87 u16 max_distance_pages;88 /* maximum possible blocks for pclusters in the filesystem */89 u16 max_pclusterblks;90};91 92struct erofs_domain {93 refcount_t ref;94 struct list_head list;95 struct fscache_volume *volume;96 char *domain_id;97};98 99struct erofs_fscache {100 struct fscache_cookie *cookie;101 struct inode *inode; /* anonymous inode for the blob */102 103 /* used for share domain mode */104 struct erofs_domain *domain;105 struct list_head node;106 refcount_t ref;107 char *name;108};109 110struct erofs_xattr_prefix_item {111 struct erofs_xattr_long_prefix *prefix;112 u8 infix_len;113};114 115struct erofs_sb_info {116 struct erofs_mount_opts opt; /* options */117#ifdef CONFIG_EROFS_FS_ZIP118 /* list for all registered superblocks, mainly for shrinker */119 struct list_head list;120 struct mutex umount_mutex;121 122 /* managed XArray arranged in physical block number */123 struct xarray managed_pslots;124 125 unsigned int shrinker_run_no;126 u16 available_compr_algs;127 128 /* pseudo inode to manage cached pages */129 struct inode *managed_cache;130 131 struct erofs_sb_lz4_info lz4;132#endif /* CONFIG_EROFS_FS_ZIP */133 struct file *fdev;134 struct inode *packed_inode;135 struct erofs_dev_context *devs;136 struct dax_device *dax_dev;137 u64 dax_part_off;138 u64 total_blocks;139 u32 primarydevice_blocks;140 141 u32 meta_blkaddr;142#ifdef CONFIG_EROFS_FS_XATTR143 u32 xattr_blkaddr;144 u32 xattr_prefix_start;145 u8 xattr_prefix_count;146 struct erofs_xattr_prefix_item *xattr_prefixes;147 unsigned int xattr_filter_reserved;148#endif149 u16 device_id_mask; /* valid bits of device id to be used */150 151 unsigned char islotbits; /* inode slot unit size in bit shift */152 unsigned char blkszbits; /* filesystem block size in bit shift */153 154 u32 sb_size; /* total superblock size */155 u32 build_time_nsec;156 u64 build_time;157 158 /* what we really care is nid, rather than ino.. */159 erofs_nid_t root_nid;160 erofs_nid_t packed_nid;161 /* used for statfs, f_files - f_favail */162 u64 inos;163 164 u8 uuid[16]; /* 128-bit uuid for volume */165 u8 volume_name[16]; /* volume name */166 u32 feature_compat;167 u32 feature_incompat;168 169 /* sysfs support */170 struct kobject s_kobj; /* /sys/fs/erofs/<devname> */171 struct completion s_kobj_unregister;172 173 /* fscache support */174 struct fscache_volume *volume;175 struct erofs_fscache *s_fscache;176 struct erofs_domain *domain;177 char *fsid;178 char *domain_id;179};180 181#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)182#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)183 184/* Mount flags set via mount options or defaults */185#define EROFS_MOUNT_XATTR_USER 0x00000010186#define EROFS_MOUNT_POSIX_ACL 0x00000020187#define EROFS_MOUNT_DAX_ALWAYS 0x00000040188#define EROFS_MOUNT_DAX_NEVER 0x00000080189 190#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option)191#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option)192#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option)193 194static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi)195{196 return IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && sbi->fdev;197}198 199static inline bool erofs_is_fscache_mode(struct super_block *sb)200{201 return IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) &&202 !erofs_is_fileio_mode(EROFS_SB(sb)) && !sb->s_bdev;203}204 205enum {206 EROFS_ZIP_CACHE_DISABLED,207 EROFS_ZIP_CACHE_READAHEAD,208 EROFS_ZIP_CACHE_READAROUND209};210 211/* basic unit of the workstation of a super_block */212struct erofs_workgroup {213 pgoff_t index;214 struct lockref lockref;215};216 217enum erofs_kmap_type {218 EROFS_NO_KMAP, /* don't map the buffer */219 EROFS_KMAP, /* use kmap_local_page() to map the buffer */220};221 222struct erofs_buf {223 struct address_space *mapping;224 struct page *page;225 void *base;226 enum erofs_kmap_type kmap_type;227};228#define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL })229 230#define erofs_blknr(sb, addr) ((erofs_blk_t)((addr) >> (sb)->s_blocksize_bits))231#define erofs_blkoff(sb, addr) ((addr) & ((sb)->s_blocksize - 1))232#define erofs_pos(sb, blk) ((erofs_off_t)(blk) << (sb)->s_blocksize_bits)233#define erofs_iblks(i) (round_up((i)->i_size, i_blocksize(i)) >> (i)->i_blkbits)234 235#define EROFS_FEATURE_FUNCS(name, compat, feature) \236static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \237{ \238 return sbi->feature_##compat & EROFS_FEATURE_##feature; \239}240 241EROFS_FEATURE_FUNCS(zero_padding, incompat, INCOMPAT_ZERO_PADDING)242EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)243EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)244EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)245EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)246EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)247EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)248EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)249EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)250EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)251EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)252EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)253 254/* atomic flag definitions */255#define EROFS_I_EA_INITED_BIT 0256#define EROFS_I_Z_INITED_BIT 1257 258/* bitlock definitions (arranged in reverse order) */259#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)260#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)261 262struct erofs_inode {263 erofs_nid_t nid;264 265 /* atomic flags (including bitlocks) */266 unsigned long flags;267 268 unsigned char datalayout;269 unsigned char inode_isize;270 unsigned int xattr_isize;271 272 unsigned int xattr_name_filter;273 unsigned int xattr_shared_count;274 unsigned int *xattr_shared_xattrs;275 276 union {277 erofs_blk_t raw_blkaddr;278 struct {279 unsigned short chunkformat;280 unsigned char chunkbits;281 };282#ifdef CONFIG_EROFS_FS_ZIP283 struct {284 unsigned short z_advise;285 unsigned char z_algorithmtype[2];286 unsigned char z_logical_clusterbits;287 unsigned long z_tailextent_headlcn;288 union {289 struct {290 erofs_off_t z_idataoff;291 unsigned short z_idata_size;292 };293 erofs_off_t z_fragmentoff;294 };295 };296#endif /* CONFIG_EROFS_FS_ZIP */297 };298 /* the corresponding vfs inode */299 struct inode vfs_inode;300};301 302#define EROFS_I(ptr) container_of(ptr, struct erofs_inode, vfs_inode)303 304static inline erofs_off_t erofs_iloc(struct inode *inode)305{306 struct erofs_sb_info *sbi = EROFS_I_SB(inode);307 308 return erofs_pos(inode->i_sb, sbi->meta_blkaddr) +309 (EROFS_I(inode)->nid << sbi->islotbits);310}311 312static inline unsigned int erofs_inode_version(unsigned int ifmt)313{314 return (ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK;315}316 317static inline unsigned int erofs_inode_datalayout(unsigned int ifmt)318{319 return (ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK;320}321 322/* reclaiming is never triggered when allocating new folios. */323static inline struct folio *erofs_grab_folio_nowait(struct address_space *as,324 pgoff_t index)325{326 return __filemap_get_folio(as, index,327 FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,328 readahead_gfp_mask(as) & ~__GFP_RECLAIM);329}330 331/* Has a disk mapping */332#define EROFS_MAP_MAPPED 0x0001333/* Located in metadata (could be copied from bd_inode) */334#define EROFS_MAP_META 0x0002335/* The extent is encoded */336#define EROFS_MAP_ENCODED 0x0004337/* The length of extent is full */338#define EROFS_MAP_FULL_MAPPED 0x0008339/* Located in the special packed inode */340#define EROFS_MAP_FRAGMENT 0x0010341/* The extent refers to partial decompressed data */342#define EROFS_MAP_PARTIAL_REF 0x0020343 344struct erofs_map_blocks {345 struct erofs_buf buf;346 347 erofs_off_t m_pa, m_la;348 u64 m_plen, m_llen;349 350 unsigned short m_deviceid;351 char m_algorithmformat;352 unsigned int m_flags;353};354 355/*356 * Used to get the exact decompressed length, e.g. fiemap (consider lookback357 * approach instead if possible since it's more metadata lightweight.)358 */359#define EROFS_GET_BLOCKS_FIEMAP 0x0001360/* Used to map the whole extent if non-negligible data is requested for LZMA */361#define EROFS_GET_BLOCKS_READMORE 0x0002362/* Used to map tail extent for tailpacking inline or fragment pcluster */363#define EROFS_GET_BLOCKS_FINDTAIL 0x0004364 365enum {366 Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,367 Z_EROFS_COMPRESSION_INTERLACED,368 Z_EROFS_COMPRESSION_RUNTIME_MAX369};370 371struct erofs_map_dev {372 struct erofs_fscache *m_fscache;373 struct block_device *m_bdev;374 struct dax_device *m_daxdev;375 struct file *m_fp;376 u64 m_dax_part_off;377 378 erofs_off_t m_pa;379 unsigned int m_deviceid;380};381 382extern const struct super_operations erofs_sops;383 384extern const struct address_space_operations erofs_aops;385extern const struct address_space_operations erofs_fileio_aops;386extern const struct address_space_operations z_erofs_aops;387extern const struct address_space_operations erofs_fscache_access_aops;388 389extern const struct inode_operations erofs_generic_iops;390extern const struct inode_operations erofs_symlink_iops;391extern const struct inode_operations erofs_fast_symlink_iops;392extern const struct inode_operations erofs_dir_iops;393 394extern const struct file_operations erofs_file_fops;395extern const struct file_operations erofs_dir_fops;396 397extern const struct iomap_ops z_erofs_iomap_report_ops;398 399/* flags for erofs_fscache_register_cookie() */400#define EROFS_REG_COOKIE_SHARE 0x0001401#define EROFS_REG_COOKIE_NEED_NOEXIST 0x0002402 403void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf,404 erofs_off_t *offset, int *lengthp);405void erofs_unmap_metabuf(struct erofs_buf *buf);406void erofs_put_metabuf(struct erofs_buf *buf);407void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset,408 enum erofs_kmap_type type);409void erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb);410void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,411 erofs_off_t offset, enum erofs_kmap_type type);412int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev);413int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,414 u64 start, u64 len);415int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map);416void erofs_onlinefolio_init(struct folio *folio);417void erofs_onlinefolio_split(struct folio *folio);418void erofs_onlinefolio_end(struct folio *folio, int err);419struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid);420int erofs_getattr(struct mnt_idmap *idmap, const struct path *path,421 struct kstat *stat, u32 request_mask,422 unsigned int query_flags);423int erofs_namei(struct inode *dir, const struct qstr *name,424 erofs_nid_t *nid, unsigned int *d_type);425 426static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)427{428 int retried = 0;429 430 while (1) {431 void *p = vm_map_ram(pages, count, -1);432 433 /* retry two more times (totally 3 times) */434 if (p || ++retried >= 3)435 return p;436 vm_unmap_aliases();437 }438 return NULL;439}440 441int erofs_register_sysfs(struct super_block *sb);442void erofs_unregister_sysfs(struct super_block *sb);443int __init erofs_init_sysfs(void);444void erofs_exit_sysfs(void);445 446struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv);447static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)448{449 return __erofs_allocpage(pagepool, gfp, false);450}451static inline void erofs_pagepool_add(struct page **pagepool, struct page *page)452{453 set_page_private(page, (unsigned long)*pagepool);454 *pagepool = page;455}456void erofs_release_pages(struct page **pagepool);457 458#ifdef CONFIG_EROFS_FS_ZIP459void erofs_workgroup_put(struct erofs_workgroup *grp);460struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,461 pgoff_t index);462struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,463 struct erofs_workgroup *grp);464void erofs_workgroup_free_rcu(struct erofs_workgroup *grp);465void erofs_shrinker_register(struct super_block *sb);466void erofs_shrinker_unregister(struct super_block *sb);467int __init erofs_init_shrinker(void);468void erofs_exit_shrinker(void);469int __init z_erofs_init_subsystem(void);470void z_erofs_exit_subsystem(void);471int erofs_try_to_free_all_cached_folios(struct erofs_sb_info *sbi,472 struct erofs_workgroup *egrp);473int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,474 int flags);475void *z_erofs_get_gbuf(unsigned int requiredpages);476void z_erofs_put_gbuf(void *ptr);477int z_erofs_gbuf_growsize(unsigned int nrpages);478int __init z_erofs_gbuf_init(void);479void z_erofs_gbuf_exit(void);480int erofs_init_managed_cache(struct super_block *sb);481int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb);482#else483static inline void erofs_shrinker_register(struct super_block *sb) {}484static inline void erofs_shrinker_unregister(struct super_block *sb) {}485static inline int erofs_init_shrinker(void) { return 0; }486static inline void erofs_exit_shrinker(void) {}487static inline int z_erofs_init_subsystem(void) { return 0; }488static inline void z_erofs_exit_subsystem(void) {}489static inline int erofs_init_managed_cache(struct super_block *sb) { return 0; }490#endif /* !CONFIG_EROFS_FS_ZIP */491 492#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE493struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev);494void erofs_fileio_submit_bio(struct bio *bio);495#else496static inline struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) { return NULL; }497static inline void erofs_fileio_submit_bio(struct bio *bio) {}498#endif499 500#ifdef CONFIG_EROFS_FS_ONDEMAND501int erofs_fscache_register_fs(struct super_block *sb);502void erofs_fscache_unregister_fs(struct super_block *sb);503 504struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,505 char *name, unsigned int flags);506void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache);507struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev);508void erofs_fscache_submit_bio(struct bio *bio);509#else510static inline int erofs_fscache_register_fs(struct super_block *sb)511{512 return -EOPNOTSUPP;513}514static inline void erofs_fscache_unregister_fs(struct super_block *sb) {}515 516static inline517struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,518 char *name, unsigned int flags)519{520 return ERR_PTR(-EOPNOTSUPP);521}522 523static inline void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache)524{525}526static inline struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev) { return NULL; }527static inline void erofs_fscache_submit_bio(struct bio *bio) {}528#endif529 530#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */531 532#endif /* __EROFS_INTERNAL_H */533