diff options
| author | Kumar Kartikeya Dwivedi <memxor@gmail.com> | 2026-09-04 10:43:19 +0200 |
|---|---|---|
| committer | Alexei Starovoitov <ast@kernel.org> | 2026-09-04 07:58:36 -0700 |
| commit | 6668ed271eaefaa63e686bdfbedaeb7b8e492722 (patch) | |
| tree | 58e8288fead484f6755ad0815fc74c3742663761 /fs/erofs/internal.h | |
| download | linux-stable-6668ed271eaefaa63e686bdfbedaeb7b8e492722.tar.gz linux-stable-6668ed271eaefaa63e686bdfbedaeb7b8e492722.zip | |
selftests/bpf: Reject graph kptr use after RCU unlockgrafted
Add a sleepable verifier test that loads a graph-node local kptr in an
explicit RCU read-side critical section, then passes its node to
bpf_rbtree_remove() after the section ends.
Before the verifier fix, the stale NON_OWN_REF flag makes the node look like
a live borrowed reference and the program is accepted. After the fix, the
pointer is demoted without NON_OWN_REF and the graph kfunc argument is
rejected.
Also exercise a graph kptr loaded while a spin lock provides implicit RCU
protection. The pointer must be invalidated when the lock is released, which
guards the required ordering between non-owning-reference invalidation and
RCU demotion.
Update the existing fault-protected load test state description. The
post-unlock pointer no longer carries NON_OWN_REF, but remains readable
because the load is rewritten to use BPF_PROBE_MEM.
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20260904084325.52250-7-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Diffstat (limited to 'fs/erofs/internal.h')
| -rw-r--r-- | fs/erofs/internal.h | 535 |
1 files changed, 535 insertions, 0 deletions
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h new file mode 100644 index 000000000..57bd21859 --- /dev/null +++ b/fs/erofs/internal.h @@ -0,0 +1,535 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_INTERNAL_H +#define __EROFS_INTERNAL_H + +#include <linux/fs.h> +#include <linux/dax.h> +#include <linux/dcache.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/pagemap.h> +#include <linux/bio.h> +#include <linux/magic.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/iomap.h> +#include "erofs_fs.h" + +__printf(2, 3) void _erofs_printk(struct super_block *sb, const char *fmt, ...); +#define erofs_err(sb, fmt, ...) \ + _erofs_printk(sb, KERN_ERR fmt "\n", ##__VA_ARGS__) +#define erofs_warn(sb, fmt, ...) \ + _erofs_printk(sb, KERN_WARNING fmt "\n", ##__VA_ARGS__) +#define erofs_info(sb, fmt, ...) \ + _erofs_printk(sb, KERN_INFO fmt "\n", ##__VA_ARGS__) + +#ifdef CONFIG_EROFS_FS_DEBUG +#define DBG_BUGON BUG_ON +#else +#define DBG_BUGON(x) ((void)(x)) +#endif /* !CONFIG_EROFS_FS_DEBUG */ + +/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */ +#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1 + +typedef u64 erofs_nid_t; +typedef u64 erofs_off_t; +typedef u64 erofs_blk_t; + +struct erofs_device_info { + char *path; + struct file *file; + struct dax_device *dax_dev; + u64 fsoff, dax_part_off; + + erofs_blk_t blocks; + erofs_blk_t uniaddr; +}; + +enum { + EROFS_SYNC_DECOMPRESS_AUTO, + EROFS_SYNC_DECOMPRESS_FORCE_ON, + EROFS_SYNC_DECOMPRESS_FORCE_OFF +}; + +struct erofs_mount_opts { + /* current strategy of how to use managed cache */ + unsigned char cache_strategy; + unsigned int mount_opt; +}; + +struct erofs_dev_context { + struct idr tree; + struct rw_semaphore rwsem; + + unsigned int extra_devices; + bool flatdev; +}; + +/* all filesystem-wide lz4 configurations */ +struct erofs_sb_lz4_info { + /* # of pages needed for EROFS lz4 rolling decompression */ + u16 max_distance_pages; + /* maximum possible blocks for pclusters in the filesystem */ + u16 max_pclusterblks; +}; + +struct erofs_xattr_prefix_item { + struct erofs_xattr_long_prefix *prefix; + u8 infix_len; +}; + +struct erofs_sb_info { + struct erofs_device_info dif0; + struct erofs_mount_opts opt; /* options */ +#ifdef CONFIG_EROFS_FS_ZIP + /* list for all registered superblocks, mainly for shrinker */ + struct list_head list; + struct mutex umount_mutex; + + /* managed XArray arranged in physical block number */ + struct xarray managed_pslots; + + unsigned int sync_decompress; /* strategy for sync decompression */ + unsigned int shrinker_run_no; + + /* pseudo inode to manage cached pages */ + struct inode *managed_cache; + + struct erofs_sb_lz4_info lz4; +#endif /* CONFIG_EROFS_FS_ZIP */ + struct inode *packed_inode; + struct inode *metabox_inode; + struct erofs_dev_context *devs; + u64 total_blocks; + + u32 meta_blkaddr; +#ifdef CONFIG_EROFS_FS_XATTR + u32 xattr_blkaddr; + u32 xattr_prefix_start; + u8 xattr_prefix_count; + u8 ishare_xattr_prefix_id; + struct erofs_xattr_prefix_item *xattr_prefixes; + unsigned int xattr_filter_reserved; +#endif + u16 device_id_mask; /* valid bits of device id to be used */ + + unsigned char islotbits; /* inode slot unit size in bit shift */ + unsigned char blkszbits; /* filesystem block size in bit shift */ + + u32 sb_size; /* total superblock size */ + u32 fixed_nsec; + s64 epoch; + + /* what we really care is nid, rather than ino.. */ + erofs_nid_t root_nid; + erofs_nid_t packed_nid; + erofs_nid_t metabox_nid; + /* used for statfs, f_files - f_favail */ + u64 inos; + + char *volume_name; + u32 feature_compat; + u32 feature_incompat; + u16 available_compr_algs; + + /* sysfs support */ + struct kobject s_kobj; /* /sys/fs/erofs/<devname> */ + struct completion s_kobj_unregister; + erofs_off_t dir_ra_bytes; + + char *domain_id; +}; + +#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) +#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info) + +/* Mount flags set via mount options or defaults */ +#define EROFS_MOUNT_XATTR_USER 0x00000010 +#define EROFS_MOUNT_POSIX_ACL 0x00000020 +#define EROFS_MOUNT_DAX_ALWAYS 0x00000040 +#define EROFS_MOUNT_DAX_NEVER 0x00000080 +#define EROFS_MOUNT_DIRECT_IO 0x00000100 +#define EROFS_MOUNT_INODE_SHARE 0x00000200 + +#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option) + +static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi) +{ + return IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && sbi->dif0.file; +} + +extern struct file_system_type erofs_anon_fs_type; + +enum { + EROFS_ZIP_CACHE_DISABLED, + EROFS_ZIP_CACHE_READAHEAD, + EROFS_ZIP_CACHE_READAROUND +}; + +struct erofs_buf { + struct address_space *mapping; + struct file *file; + u64 off; + struct page *page; + void *base; +}; +#define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL }) + +#define erofs_blknr(sb, pos) ((erofs_blk_t)((pos) >> (sb)->s_blocksize_bits)) +#define erofs_blkoff(sb, pos) ((pos) & ((sb)->s_blocksize - 1)) +#define erofs_pos(sb, blk) ((erofs_off_t)(blk) << (sb)->s_blocksize_bits) +#define erofs_iblks(i) (round_up((i)->i_size, i_blocksize(i)) >> (i)->i_blkbits) + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ +static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \ +{ \ + return sbi->feature_##compat & EROFS_FEATURE_##feature; \ +} + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) +EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) +EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES) +EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT) +EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) +EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) +EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX) +EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX) +EROFS_FEATURE_FUNCS(ishare_xattrs, compat, COMPAT_ISHARE_XATTRS) + +static inline u64 erofs_nid_to_ino64(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + if (!erofs_sb_has_metabox(sbi)) + return nid; + + /* + * When metadata compression is enabled, avoid generating excessively + * large inode numbers for metadata-compressed inodes. Shift NIDs in + * the 31-62 bit range left by one and move the metabox flag to bit 31. + * + * Note: on-disk NIDs remain unchanged as they are primarily used for + * compatibility with non-LFS 32-bit applications. + */ + return ((nid << 1) & GENMASK_ULL(63, 32)) | (nid & GENMASK(30, 0)) | + ((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31); +} + +/* atomic flag definitions */ +#define EROFS_I_EA_INITED_BIT 0 +#define EROFS_I_Z_INITED_BIT 1 + +/* bitlock definitions (arranged in reverse order) */ +#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1) +#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2) + +/* default readahead size of directories */ +#define EROFS_DIR_RA_BYTES 16384 + +struct erofs_inode_fingerprint { + u8 *opaque; + int size; +}; + +struct erofs_inode { + erofs_nid_t nid; + + /* atomic flags (including bitlocks) */ + unsigned long flags; + + unsigned char datalayout; + unsigned char inode_isize; + bool dot_omitted; + unsigned int xattr_isize; + + unsigned int xattr_name_filter; + unsigned int xattr_shared_count; + unsigned int *xattr_shared_xattrs; + + union { + erofs_blk_t startblk; + struct { + unsigned short chunkformat; + unsigned char chunkbits; + }; +#ifdef CONFIG_EROFS_FS_ZIP + struct { + unsigned short z_advise; + unsigned char z_algorithmtype[2]; + unsigned char z_lclusterbits; + union { + u64 z_tailextent_headlcn; + u64 z_extents; + }; + erofs_off_t z_fragmentoff; + unsigned short z_idata_size; + }; +#endif /* CONFIG_EROFS_FS_ZIP */ + }; +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE + struct list_head ishare_list; + union { + /* for each anon shared inode */ + struct { + struct erofs_inode_fingerprint fingerprint; + spinlock_t ishare_lock; + }; + /* for each real filesystem inode */ + struct dentry *sharedentry; + }; +#endif + /* the corresponding vfs inode */ + struct inode vfs_inode; +}; + +#define EROFS_I(ptr) container_of(ptr, struct erofs_inode, vfs_inode) + +static inline bool erofs_inode_in_metabox(struct inode *inode) +{ + return EROFS_I(inode)->nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT); +} + +static inline erofs_off_t erofs_iloc(struct inode *inode) +{ + struct erofs_sb_info *sbi = EROFS_I_SB(inode); + erofs_nid_t nid_lo = EROFS_I(inode)->nid & EROFS_DIRENT_NID_MASK; + + if (erofs_inode_in_metabox(inode)) + return nid_lo << sbi->islotbits; + return erofs_pos(inode->i_sb, sbi->meta_blkaddr) + + (nid_lo << sbi->islotbits); +} + +static inline unsigned int erofs_inode_version(unsigned int ifmt) +{ + return (ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK; +} + +static inline unsigned int erofs_inode_datalayout(unsigned int ifmt) +{ + return (ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK; +} + +/* reclaiming is never triggered when allocating new folios. */ +static inline struct folio *erofs_grab_folio_nowait(struct address_space *as, + pgoff_t index) +{ + return __filemap_get_folio(as, index, + FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT, + readahead_gfp_mask(as) & ~__GFP_RECLAIM); +} + +/* Allocated on disk at @m_pa (e.g. NOT a fragment extent) */ +#define EROFS_MAP_MAPPED 0x0001 +/* Located in metadata (could be copied from bd_inode) */ +#define EROFS_MAP_META 0x0002 +/* @m_llen may be truncated by the runtime compared to the on-disk record */ +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +/* The on-disk @m_llen may cover only part of the encoded data */ +#define EROFS_MAP_PARTIAL_REF 0x0008 +/* Located in the special packed inode */ +#define EROFS_MAP_FRAGMENT 0x0010 +/* The encoded on-disk data will be fully handled (decompressed) */ +#define EROFS_MAP_FULL(f) (!((f) & (EROFS_MAP_PARTIAL_MAPPED | \ + EROFS_MAP_PARTIAL_REF))) + +struct erofs_map_blocks { + struct erofs_buf buf; + + erofs_off_t m_pa, m_la; + u64 m_plen, m_llen; + + unsigned short m_deviceid; + char m_algorithmformat; + unsigned int m_flags; +}; + +/* + * Used to get the exact decompressed length, e.g. fiemap (consider lookback + * approach instead if possible since it's more metadata lightweight.) + */ +#define EROFS_GET_BLOCKS_FIEMAP 0x0001 +/* Used to map the whole extent if non-negligible data is requested for LZMA */ +#define EROFS_GET_BLOCKS_READMORE 0x0002 +/* Used to map tail extent for tailpacking inline or fragment pcluster */ +#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +struct erofs_map_dev { + struct super_block *m_sb; + struct erofs_device_info *m_dif; + struct block_device *m_bdev; + + erofs_off_t m_pa; + unsigned int m_deviceid; +}; + +extern const struct super_operations erofs_sops; +extern const struct address_space_operations erofs_aops; +extern const struct address_space_operations erofs_fileio_aops; +extern const struct address_space_operations z_erofs_aops; + +extern const struct inode_operations erofs_generic_iops; +extern const struct inode_operations erofs_symlink_iops; +extern const struct inode_operations erofs_fast_symlink_iops; +extern const struct inode_operations erofs_dir_iops; + +extern const struct file_operations erofs_file_fops; +extern const struct file_operations erofs_dir_fops; +extern const struct file_operations erofs_ishare_fops; + +extern const struct iomap_ops z_erofs_iomap_report_ops; + +void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf, + erofs_off_t *offset, int *lengthp); +void erofs_unmap_metabuf(struct erofs_buf *buf); +void erofs_put_metabuf(struct erofs_buf *buf); +void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap); +int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, + bool in_metabox); +void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb, + erofs_off_t offset, bool in_metabox); +int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev); +int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); +int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map); +void erofs_onlinefolio_init(struct folio *folio); +void erofs_onlinefolio_split(struct folio *folio); +void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty); +struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid); +int erofs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags); +int erofs_namei(struct inode *dir, const struct qstr *name, + erofs_nid_t *nid, unsigned int *d_type); + +static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count) +{ + int retried = 0; + + while (1) { + void *p = vm_map_ram(pages, count, -1); + + /* retry two more times (totally 3 times) */ + if (p || ++retried >= 3) + return p; + vm_unmap_aliases(); + } + return NULL; +} + +static inline const struct address_space_operations * +erofs_get_aops(struct inode *realinode) +{ + if (erofs_inode_is_data_compressed(EROFS_I(realinode)->datalayout)) { + if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) + return ERR_PTR(-EOPNOTSUPP); + DO_ONCE_LITE_IF(realinode->i_blkbits != PAGE_SHIFT, + erofs_info, realinode->i_sb, + "EXPERIMENTAL EROFS subpage compressed block support in use. Use at your own risk!"); + return &z_erofs_aops; + } + if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && + erofs_is_fileio_mode(EROFS_SB(realinode->i_sb))) + return &erofs_fileio_aops; + return &erofs_aops; +} + +int erofs_register_sysfs(struct super_block *sb); +void erofs_unregister_sysfs(struct super_block *sb); +int __init erofs_init_sysfs(void); +void erofs_exit_sysfs(void); + +struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv); +static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp) +{ + return __erofs_allocpage(pagepool, gfp, false); +} +static inline void erofs_pagepool_add(struct page **pagepool, struct page *page) +{ + set_page_private(page, (unsigned long)*pagepool); + *pagepool = page; +} +void erofs_release_pages(struct page **pagepool); + +#ifdef CONFIG_EROFS_FS_ZIP +#define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping) + +extern atomic_long_t erofs_global_shrink_cnt; +void erofs_shrinker_register(struct super_block *sb); +void erofs_shrinker_unregister(struct super_block *sb); +int __init erofs_init_shrinker(void); +void erofs_exit_shrinker(void); +int __init z_erofs_init_subsystem(void); +void z_erofs_exit_subsystem(void); +int z_erofs_init_super(struct super_block *sb); +unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi, + unsigned long nr_shrink); +int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, + int flags); +void *z_erofs_get_gbuf(unsigned int requiredpages); +void z_erofs_put_gbuf(void *ptr); +int z_erofs_gbuf_growsize(unsigned int nrpages); +int __init z_erofs_gbuf_init(void); +void z_erofs_gbuf_exit(void); +#else +static inline void erofs_shrinker_register(struct super_block *sb) {} +static inline void erofs_shrinker_unregister(struct super_block *sb) {} +static inline int erofs_init_shrinker(void) { return 0; } +static inline void erofs_exit_shrinker(void) {} +static inline int z_erofs_init_subsystem(void) { return 0; } +static inline void z_erofs_exit_subsystem(void) {} +static inline int z_erofs_init_super(struct super_block *sb) { return 0; } +#endif /* !CONFIG_EROFS_FS_ZIP */ +int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb); + +#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE +struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev); +void erofs_fileio_submit_bio(struct bio *bio); +#else +static inline struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) { return NULL; } +static inline void erofs_fileio_submit_bio(struct bio *bio) {} +#endif + +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE +int __init erofs_init_ishare(void); +void erofs_exit_ishare(void); +bool erofs_ishare_fill_inode(struct inode *inode); +void erofs_ishare_free_inode(struct inode *inode); +struct inode *erofs_real_inode(struct inode *inode, bool *need_iput); +#else +static inline int erofs_init_ishare(void) { return 0; } +static inline void erofs_exit_ishare(void) {} +static inline bool erofs_ishare_fill_inode(struct inode *inode) { return false; } +static inline void erofs_ishare_free_inode(struct inode *inode) {} +static inline struct inode *erofs_real_inode(struct inode *inode, bool *need_iput) +{ + *need_iput = false; + return inode; +} +#endif + +long erofs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +long erofs_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); + +#endif /* __EROFS_INTERNAL_H */ |
